diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 2199726d0e..6ac26e1a35 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -21,7 +21,7 @@ on: jobs: vlm_clickhouse: runs-on: ubuntu-latest - container: hailgenetics/hail:0.2.128 + container: python:3.11-slim-bullseye services: clickhouse: @@ -44,6 +44,7 @@ jobs: - name: Install dependencies run: | python3 -m pip install --upgrade pip wheel + pip install -r vlm/requirements.txt pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Settings and Data run: python3 vlm/setup_clickhouse_test_data.py clickhouse 8123 clickhouse_test_user clickhouse_test_password @@ -57,5 +58,5 @@ jobs: export CLICKHOUSE_VLM_PASSWORD=vlm_test_password export CLICKHOUSE_DATABASE=test_seqr coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py" -m pytest vlm/ - coverage report --fail-under=95 + coverage report -m --fail-under=95 diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index fc11948cdc..74d4c11442 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -12,7 +12,8 @@ from clickhouse_search.models.gt_stats_models import ProjectGtStatsSnvIndel, \ ProjectsToGtStatsGRCh37SnvIndel, ProjectsToGtStatsSnvIndel, ProjectsToGtStatsMito, ProjectsToGtStatsSv, \ GtStatsDictGRCh37SnvIndel, GtStatsDictSnvIndel, GtStatsDictMito, GtStatsDictSv -from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict +from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict, IndividualMetadataDict, DiscoveryVariantDict, \ + ExcludedVariantDict from clickhouse_search.models.reference_data_models import ClinvarMvSnvIndel, ClinvarSearchMvSnvIndel, ClinvarMvMito, \ ClinvarSearchMvMito, ClinvarMvGRCh37SnvIndel, ClinvarSearchMvGRCh37SnvIndel, HgmdMv, HgmdSearchMv, \ DbnsfpSnvIndelMv, DbnsfpSnvIndelDict, EigenMv, EigenDict, SpliceAiMv, SpliceAiDict, GnomadNonCodingConstraintDict, \ @@ -33,12 +34,12 @@ VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GRCH37_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, \ SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4, SV_GENE_COUNTS, NEW_SV_FILTER, GCNV_VARIANT1, GCNV_VARIANT2, \ GCNV_VARIANT3, GCNV_VARIANT4, GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_GENE_COUNTS, \ - MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, \ + MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, DISCOVERY_VARIANT, \ MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, FAMILY_1_VARIANT, EXPORT_DATA, SPLIT_FAMILY_EXPORT_DATA, \ DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, \ format_cached_variant from reference_data.models import Omim -from seqr.models import Project, Family, Dataset, VariantSearch, VariantSearchResults, SavedVariant, Individual +from seqr.models import Project, Family, Dataset, VariantSearch, VariantSearchResults, Individual from seqr.views.apis.data_manager_api import trigger_delete_project from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, \ SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \ @@ -81,8 +82,8 @@ def tearDownClass(cls): @classmethod def setUpTestData(cls): - AffectedDict.reload() - SexDict.reload() + for postgres_dict in [AffectedDict, SexDict, IndividualMetadataDict, DiscoveryVariantDict, ExcludedVariantDict]: + postgres_dict.reload() for view in [ ProjectsToGtStatsGRCh37SnvIndel, ProjectsToGtStatsSnvIndel, ProjectsToGtStatsMito, ProjectsToGtStatsSv, ClinvarMvSnvIndel, ClinvarSearchMvSnvIndel, ClinvarMvMito, ClinvarSearchMvMito, ClinvarMvGRCh37SnvIndel, @@ -108,7 +109,7 @@ def setUpTestData(cls): class ClickhouseSearchTests(ClickhouseSearchTestCase): databases = '__all__' - fixtures = ['users', 'social_auth', '1kg_project', 'variant_searches', 'reference_data', 'clickhouse_search', 'clickhouse_transcripts'] + fixtures = ['users', 'social_auth', '1kg_project', 'variant_searches', 'reference_data', 'clickhouse_discovery_variants', 'clickhouse_search', 'clickhouse_transcripts'] def setUp(self): self.MOCK_CACHE = {} @@ -164,9 +165,10 @@ def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter return response, search_hash, search_body - def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', is_37=False, skip_cache_check=False, response_search=None, project_families=None, additional_response=None, export_data=None, cache_sort=None, **kwargs): + def _assert_expected_search(self, expected_results, results_page=None, gene_counts=None, cached_variant_fields=None, sort='xpos', is_37=False, skip_cache_check=False, response_search=None, project_families=None, searched_project_families=None, additional_response=None, export_data=None, cache_sort=None, **kwargs): response, search_hash, search_body = self._execute_search(project_families=project_families, sort=sort, **kwargs) self.assertEqual(response.status_code, 200) + project_families = searched_project_families or project_families expected_response = { 'searchedVariantIds': [], 'variantsById': {}, @@ -394,17 +396,23 @@ def test_all_project_search(self): ) request_body['unsolvedFamiliesOnly'] = True - project_families[0]['familyGuids'].remove('F000007_7') - project_families[0]['familyGuids'].remove('F000010_10') + searched_project_families = [{ + **project_families[0], + 'familyGuids': [guid for guid in project_families[0]['familyGuids'] if guid not in {'F000007_7', 'F000010_10'}], + }] self._assert_expected_search( - results, request_body=request_body, project_families=project_families, additional_response=additional_response, locus=locus, + [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, + GCNV_VARIANT4, FAMILY_1_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], + request_body=request_body, project_families=project_families, searched_project_families=searched_project_families, + additional_response=additional_response, locus=locus, ) request_body['trioFamiliesOnly'] = True self._assert_expected_search( [VARIANT1, VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], - request_body=request_body, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, locus=locus, + request_body=request_body, project_families=project_families, searched_project_families=SINGLE_FAMILY_PROJECT_FAMILIES, + locus=locus, ) def test_both_sample_types_search(self): @@ -423,6 +431,7 @@ def test_both_sample_types_search(self): dataset.sample_type = 'WGS' dataset.save() dataset.active_individuals.add(4) + self.maxDiff = None # Variant 1 is de novo in exome but inherited and homozygous in genome. # Variant 2 is inherited and homozygous in exome and de novo and homozygous in genome, so it fails de-novo inheritance when parental data is missing in genome. @@ -439,7 +448,6 @@ def test_both_sample_types_search(self): inheritance_mode='any_affected', quality_filter={'min_gq': 40, 'min_qs': 20}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) - self.maxDiff = None self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1], inheritance_mode='de_novo', quality_filter=None, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, @@ -519,8 +527,8 @@ def test_inheritance_filter(self): ) inheritance_mode = 'x_linked_recessive' - self._assert_expected_search([], inheritance_mode=inheritance_mode) - # self._assert_expected_search([], inheritance_mode=inheritance_mode, sample_data=SV_WGS_SAMPLE_DATA_WITH_SEX) + self._assert_expected_search([], inheritance_mode=inheritance_mode, export_data=[EXPORT_DATA[0][:24]]) + self._assert_expected_search([], inheritance_mode=inheritance_mode, inheritance_filter={'allowNoCall': True}) inheritance_mode = 'homozygous_recessive' self._assert_expected_search( @@ -589,6 +597,11 @@ def test_inheritance_filter(self): ], project_families=SV_PROJECT_FAMILIES, ) + self._assert_expected_search( + [], inheritance_mode=inheritance_mode, project_families=MULTI_PROJECT_PROJECT_FAMILIES, + **COMP_HET_ALL_PASS_FILTERS, locus={'rawItems': 'chrX:1-100000000'}, + ) + inheritance_mode = 'recessive' self._assert_expected_search( [PROJECT_2_VARIANT1, VARIANT2, [VARIANT3, VARIANT4], MITO_VARIANT3], inheritance_mode=inheritance_mode, gene_counts={ @@ -1040,14 +1053,13 @@ def test_variant_lookup(self, mock_liftover): {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WES'}, {'ab': 1.0, 'dp': 6, 'gq': 16, 'numAlt': 2, 'filters': [], 'sampleType': 'WGS'}, ], - 'I0_F2_1-10439-AC-A': {'ab': 0.531, 'dp': 27, 'gq': 87, 'numAlt': 1, 'filters': [], - 'sampleType': 'WGS'}, + 'I0_F2_1-10439-AC-A': {'ab': 0.531, 'dp': 27, 'gq': 87, 'numAlt': 1, 'filters': [], 'sampleType': 'WGS'}, }, } expected_individuals = { 'I0_F0_1-10439-AC-A': { 'affected': 'N', 'familyGuid': 'F0_1-10439-AC-A', 'features': [], - 'individualGuid': 'I0_F0_1-10439-AC-A', 'sex': 'F', + 'individualGuid': 'I0_F0_1-10439-AC-A', 'sex': 'F', 'isSolved': False, 'disease': 'OMIM:615123', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', }, 'I0_F1_1-10439-AC-A': { @@ -1056,20 +1068,20 @@ def test_variant_lookup(self, mock_liftover): 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', }, 'I0_F2_1-10439-AC-A': { - 'affected': 'A', 'familyGuid': 'F2_1-10439-AC-A', 'features': [], + 'affected': 'A', 'familyGuid': 'F2_1-10439-AC-A', 'features': [], 'isSolved': True, 'disease': '', 'individualGuid': 'I0_F2_1-10439-AC-A', 'sex': 'F', 'vlmContactEmail': 'vlm@broadinstitute.org', }, 'I1_F0_1-10439-AC-A': { 'affected': 'N', 'familyGuid': 'F0_1-10439-AC-A', 'features': [], - 'individualGuid': 'I1_F0_1-10439-AC-A', 'sex': 'M', + 'individualGuid': 'I1_F0_1-10439-AC-A', 'sex': 'M', 'isSolved': False, 'disease': 'OMIM:615123', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', }, 'I2_F0_1-10439-AC-A': { 'affected': 'A', 'familyGuid': 'F0_1-10439-AC-A', 'individualGuid': 'I2_F0_1-10439-AC-A', 'sex': 'X0', 'features': [{'category': 'HP:0000707', 'label': 'Morphological abnormality of the central nervous system', 'id': 'HP:0002011'}, {'category': 'HP:0001626', 'label': 'Arrhythmia', 'id': 'HP:0011675'}], - 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', 'isSolved': False, 'disease': 'OMIM:615123', }, } cache_key = 'variant_lookup_results__1-10439-AC-A__38' @@ -1119,31 +1131,34 @@ def test_variant_lookup(self, mock_liftover): 'I0_F0_phase2_DEL_chr14_4640': { 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], 'individualGuid': 'I0_F0_phase2_DEL_chr14_4640', 'sex': 'M', - 'vlmContactEmail': 'vlm@broadinstitute.org', + 'vlmContactEmail': 'vlm@broadinstitute.org', 'isSolved': True, 'disease': '', }, 'I0_F0_suffix_140608_DUP': { 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I0_F0_suffix_140608_DUP', 'sex': 'F', 'features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + 'isSolved': False, 'disease': 'OMIM:615123', }, 'I1_F0_phase2_DEL_chr14_4640': { 'affected': 'N', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], 'individualGuid': 'I1_F0_phase2_DEL_chr14_4640', 'sex': 'F', - 'vlmContactEmail': 'vlm@broadinstitute.org', + 'vlmContactEmail': 'vlm@broadinstitute.org', 'isSolved': True, 'disease': '', }, 'I1_F0_suffix_140608_DUP': { 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I1_F0_suffix_140608_DUP', 'sex': 'M', 'features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + 'isSolved': False, 'disease': 'OMIM:615123', }, 'I2_F0_phase2_DEL_chr14_4640': { 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], 'individualGuid': 'I2_F0_phase2_DEL_chr14_4640', 'sex': 'F', - 'vlmContactEmail': 'vlm@broadinstitute.org', + 'vlmContactEmail': 'vlm@broadinstitute.org', 'isSolved': True, 'disease': '', }, 'I2_F0_suffix_140608_DUP': { 'affected': 'A', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I2_F0_suffix_140608_DUP', 'features': [{'category': 'HP:0000707', 'label': 'Morphological abnormality of the central nervous system', 'id': 'HP:0002011'}, {'category': 'HP:0001626', 'label': 'Arrhythmia', 'id': 'HP:0011675'}], 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + 'isSolved': False, 'disease': 'OMIM:615123', }, } sv_genes = { @@ -1164,17 +1179,17 @@ def test_variant_lookup(self, mock_liftover): 'I000015_na20885': 'I0_F1_1-10439-AC-A', 'I000018_na21234': 'I0_F2_1-10439-AC-A', } - expected_individuals = { + access_expected_individuals = { individual_guid: { **{k: mock.ANY for k in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']}, **{k: v for k, v in expected_individuals[anon_individual_guid].items() - if k not in {'individualGuid', 'familyGuid', 'features', 'vlmContactEmail'}}, + if k not in {'individualGuid', 'familyGuid', 'features', 'vlmContactEmail', 'isSolved', 'disease'}}, } for individual_guid, anon_individual_guid in individual_guid_map.items() } - expected_individuals.update( + access_expected_individuals.update( {individual_guid: mock.ANY for individual_guid in ['I000019_na21987', 'I000021_na21654']}) self._assert_expected_lookup( - '1-10439-AC-A', lookup_variant, cache_key, expected_individuals=expected_individuals, + '1-10439-AC-A', lookup_variant, cache_key, expected_individuals=access_expected_individuals, project_guids=['R0001_1kg', 'R0003_test', 'R0004_non_analyst_project'], family_guids=['F000002_2', 'F000011_11', 'F000014_14'], ) @@ -1222,16 +1237,17 @@ def test_variant_lookup(self, mock_liftover): self.assertEqual(response.status_code, 404) self.assertDictEqual(response.json(), {'error': 'Variant not present in seqr'}) - self.set_cache('variant_lookup_results__1-91511686-TCA-G__38', [{ - **VARIANT1, - 'familyGenotypes': {'F000002_2': list(VARIANT1['genotypes'].values())}, - }]) + self.set_cache('variant_lookup_results__1-91511686-TCA-G__38', [ + self._cached_lookup_variant(VARIANT1), + ]) response = self.client.get(url) self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json()['variantsById'], {'1-10439-AC-A': { **VARIANT1, 'familyGuids': [], 'lookupFamilyGuids': VARIANT1['familyGuids'], + 'discoveryTagFamilies': [], + 'excludedTagFamilies': [], }}) cache_key = 'variant_lookup_results__7-143270172-A-G__37' @@ -1307,6 +1323,120 @@ def test_variant_lookup(self, mock_liftover): individual_guids=['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733'], ) + discovery_variant = { + **DISCOVERY_VARIANT, + 'discoveryTags': [], + 'familyGuids': ['F0_1-248367227-TC-T', 'F1_1-248367227-TC-T', 'F2_1-248367227-TC-T'], + 'discoveryTagFamilies': ['F0_1-248367227-TC-T', 'F1_1-248367227-TC-T', 'F2_1-248367227-TC-T'], + 'excludedTagFamilies': ['F0_1-248367227-TC-T'], + 'genotypes': { + 'I0_F0_1-248367227-TC-T': { + 'sampleType': 'WGS', 'numAlt': 1, 'dp': 49, 'gq': 99, 'ab': 0.65306, 'filters': [], + }, + 'I1_F0_1-248367227-TC-T': { + 'sampleType': 'WGS', 'numAlt': 0, 'dp': 16, 'gq': 48, 'ab': 1.0, 'filters': [], + }, + 'I0_F1_1-248367227-TC-T': { + 'sampleType': 'WGS', 'ab': 0.0, 'gq': 99, 'dp': 71, 'numAlt': 1, 'filters': [], + }, + 'I1_F1_1-248367227-TC-T': { + 'sampleType': 'WGS', 'ab': 0.55555, 'gq': 99, 'dp': 9, 'numAlt': 1, 'filters': [], + }, + 'I0_F2_1-248367227-TC-T': { + 'sampleType': 'WGS', 'numAlt': 2, 'dp': 49, 'gq': 99, 'ab': 0.0, 'filters': [], + }, + }, + } + cached_discovery_variant = { + **DISCOVERY_VARIANT, + 'discoveryFamilies': ['F000002_2', 'F000012_12', 'F000014_14'], + 'excludedTagFamilies': ['F000002_2'], + 'genotypes': { + **DISCOVERY_VARIANT['genotypes'], + 'I000017_na20889': { + 'sampleId': 'NA20889', 'sampleType': 'WGS', 'familyGuid': 'F000012_12', + 'individualGuid': 'I000017_na20889', + 'ab': 0.0, 'gq': 99, 'dp': 71, 'numAlt': 1, 'filters': [], + }, + 'I000016_na20888': { + 'sampleId': 'NA20888', 'sampleType': 'WGS', 'familyGuid': 'F000012_12', + 'individualGuid': 'I000016_na20888', + 'ab': 0.55555, 'gq': 99, 'dp': 9, 'numAlt': 1, 'filters': [], + }, + 'I000018_na21234': { + 'sampleId': 'NA21234', 'sampleType': 'WGS', 'familyGuid': 'F000014_14', 'individualGuid': 'I000018_na21234', + 'numAlt': 2, 'dp': 49, 'gq': 99, 'ab': 0.0, 'filters': [], + }, + }, + } + expected_individuals = { + 'I0_F0_1-248367227-TC-T': { + **expected_individuals['I0_F0_1-10439-AC-A'], + 'familyGuid': 'F0_1-248367227-TC-T', + 'individualGuid': 'I0_F0_1-248367227-TC-T', + }, 'I1_F0_1-248367227-TC-T': { + **expected_individuals['I2_F0_1-10439-AC-A'], + 'familyGuid': 'F0_1-248367227-TC-T', + 'individualGuid': 'I1_F0_1-248367227-TC-T', + }, 'I0_F1_1-248367227-TC-T': { + **expected_individuals['I0_F1_1-10439-AC-A'], + 'sex': 'F', + 'familyGuid': 'F1_1-248367227-TC-T', + 'individualGuid': 'I0_F1_1-248367227-TC-T', + }, 'I1_F1_1-248367227-TC-T': { + **expected_individuals['I0_F1_1-10439-AC-A'], + 'features': [], + 'familyGuid': 'F1_1-248367227-TC-T', + 'individualGuid': 'I1_F1_1-248367227-TC-T', + }, 'I0_F2_1-248367227-TC-T': { + **expected_individuals['I0_F2_1-10439-AC-A'], + 'familyGuid': 'F2_1-248367227-TC-T', + 'individualGuid': 'I0_F2_1-248367227-TC-T', + } + } + self.login_base_user() + self._assert_expected_lookup( + '1-248367227-TC-T', discovery_variant, 'variant_lookup_results__1-248367227-TC-T__38', + cached_variants=[cached_discovery_variant], expected_individuals=expected_individuals, skip_fields={ + 'variantFunctionalDataByGuid', 'variantNotesByGuid', 'variantTagsByGuid', + }, locusListsByGuid={}, + ) + + self.login_analyst_user() + discovery_variant = { + **discovery_variant, + 'familyGuids': ['F000002_2', 'F000012_12', 'F0_1-248367227-TC-T'], + 'discoveryTagFamilies': ['F0_1-248367227-TC-T'], + 'excludedTagFamilies': [], + 'genotypes': { + **cached_discovery_variant['genotypes'], + 'I0_F0_1-248367227-TC-T': discovery_variant['genotypes']['I0_F2_1-248367227-TC-T'], + }, + } + del discovery_variant['genotypes']['I000018_na21234'] + self._assert_expected_lookup( + '1-248367227-TC-T', discovery_variant, 'variant_lookup_results__1-248367227-TC-T__38', + cached_variants=[cached_discovery_variant], project_guids=['R0001_1kg', 'R0003_test'], + family_guids=['F000002_2', 'F000012_12'], expected_individuals={ + **{guid: mock.ANY for guid in [ + 'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000016_na20888', 'I000017_na20889', + 'I000020_na20870', + ]}, + 'I0_F0_1-248367227-TC-T': { + **expected_individuals['I0_F2_1-248367227-TC-T'], + 'familyGuid': 'F0_1-248367227-TC-T', + 'individualGuid': 'I0_F0_1-248367227-TC-T', + }, + }, + mmeSubmissionsByGuid={'MS000015_na20885': mock.ANY}, + savedVariantsByGuid={'SV0000002_1248367227_r0390_100': mock.ANY, 'SV0000006_1248367227_r0003_tes': mock.ANY}, + variantNotesByGuid={'VN0714935_2103343353_r0390_100': mock.ANY, 'VN0714937_2103343353_r0390_100': mock.ANY}, + variantTagsByGuid={ + 'VT1726945_2103343353_r0390_100': mock.ANY, 'VT1726961_2103343353_r0003_tes': mock.ANY, + 'VT1726970_2103343353_r0004_tes': mock.ANY, 'VT1726985_2103343353_r0390_100': mock.ANY, + }, + ) + # Test error handling when the ClickHouse sampleId cannot be mapped to any Postgres Individual self.reset_logs() Individual.objects.filter(guid='I000006_hg00733').update(individual_id='unmapped_id') @@ -1325,36 +1455,97 @@ def test_variant_lookup(self, mock_liftover): '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', }), ] - self.assert_json_logs(self.manager_user, unmapped_sample_logs) + self.assert_json_logs(self.analyst_user, unmapped_sample_logs) + # With no project access, all genotypes are returned regardless of whether a corresponding seqr individual exists self.login_base_user() self.reset_logs() - expected_individuals = {'I1_F0_7-143270172-A-G': { + expected_individuals = {'I0_F0_7-143270172-A-G': { + 'affected': 'N', + 'disease': 'OMIM:615123', + 'familyGuid': 'F0_7-143270172-A-G', + 'features': [], + 'individualGuid': 'I0_F0_7-143270172-A-G', + 'isSolved': False, + 'sex': 'F', + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, 'I1_F0_7-143270172-A-G': { 'affected': 'A', + 'disease': 'OMIM:615123', 'familyGuid': 'F0_7-143270172-A-G', 'features': [ {'category': 'HP:0000707', 'id': 'HP:0002011', 'label': 'Morphological abnormality of the central nervous system'}, - {'category': 'HP:0001626', 'id': 'HP:0011675', 'label': 'Arrhythmia'}, + {'category': 'HP:0001626', 'id': 'HP:0011675', 'label': 'Arrhythmia'}, ], 'individualGuid': 'I1_F0_7-143270172-A-G', + 'isSolved': False, 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', }} - no_access_missing_gt_variant = { + no_access_variant = { **GRCH37_VARIANT, 'familyGuids': ['F0_7-143270172-A-G'], - 'genotypes': {'I1_F0_7-143270172-A-G': { - k: v for k, v in GRCH37_VARIANT['genotypes']['I000004_hg00731'].items() + 'genotypes': {mapped_guid: { + k: v for k, v in GRCH37_VARIANT['genotypes'][guid].items() if k not in {'familyGuid', 'individualGuid', 'sampleId'} - }} + } for guid, mapped_guid in { + 'I000004_hg00731': 'I1_F0_7-143270172-A-G', 'I000006_hg00733': 'I0_F0_7-143270172-A-G', + }.items()} } self._assert_expected_lookup( - '7-143270172-A-G', no_access_missing_gt_variant, cache_key, cached_variants=[GRCH37_VARIANT], + '7-143270172-A-G', no_access_variant, cache_key, cached_variants=[GRCH37_VARIANT], genome_version='37', expected_individuals=expected_individuals, locusListsByGuid={}, skip_fields={ 'variantFunctionalDataByGuid', 'variantNotesByGuid', 'variantTagsByGuid', }, ) - self.assert_json_logs(self.no_access_user, unmapped_sample_logs) + self.assert_json_logs(self.no_access_user, unmapped_sample_logs[:1]) + + INDIVIDUAL_METADATA = { + 'I000006_hg00733': { + 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'F', 'omim_id': 615123, 'mondo_id': 'MONDO:0044970', + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', 'isSolved': False, + }, + 'I000005_hg00732': { + 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'M', 'isSolved': False, + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', 'omim_id': 615123, 'mondo_id': 'MONDO:0044970', + }, + 'I000004_hg00731': { + 'affected': 'A', 'features': '[{"id": "HP:0002011"}, {"id": "HP:0011675"}]', 'restrict_sharing': False, + 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', 'isSolved': False, + 'omim_id': 615123, 'mondo_id': 'MONDO:0044970', + }, + 'I000015_na20885': { + 'affected': 'A', 'features': '[{"id": "HP:0011675"}, {"id": "HP:0001509"}]', 'restrict_sharing': True, + 'sex': 'M', 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', + 'omim_id': 0, 'mondo_id': '', 'isSolved': False, + }, + 'I000016_na20888': { + 'affected': 'A', 'features': '', 'restrict_sharing': True, 'sex': 'M', + 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', + 'omim_id': 616126, 'mondo_id': '0008788', 'isSolved': False, + }, + 'I000017_na20889': { + 'affected': 'A', 'features': '[{"id": "HP:0011675"}, {"id": "HP:0001509"}]', 'restrict_sharing': True, + 'sex': 'F', 'vlmContactEmail': 'seqr-test@gmail.com,test@broadinstitute.org', + 'omim_id': 616126, 'mondo_id': '0008788', 'isSolved': False, + }, + 'I000018_na21234': { + 'affected': 'A', 'features': '', 'restrict_sharing': False, 'sex': 'F', 'isSolved': True, + 'vlmContactEmail': 'vlm@broadinstitute.org', 'omim_id': 0, 'mondo_id': '', + }, + 'I000019_na21987': { + 'affected': 'A', 'features': '', 'restrict_sharing': False, 'sex': 'M', 'isSolved': True, + 'vlmContactEmail': 'vlm@broadinstitute.org', 'omim_id': 0, 'mondo_id': '', + }, + 'I000021_na21654': { + 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'F', 'isSolved': True, + 'vlmContactEmail': 'vlm@broadinstitute.org', 'omim_id': 0, 'mondo_id': '', + }, + 'I000002_na19678': { + 'affected': 'N', 'features': '', 'restrict_sharing': False, 'sex': 'M', 'isSolved': False, + 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', 'omim_id': 615123, 'mondo_id': '', + }, + } def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version='38', hom_only=False, affected_only=False, project_guids=None, family_guids=None, individual_guids=None, expected_individuals=None, skip_fields=None, cached_variants=None, additional_variant=None, sample_type=None, **kwargs): url = f'{reverse(variant_lookup_handler)}?variantId={variant_id}&genomeVersion={genome_version}' @@ -1382,6 +1573,8 @@ def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version 'locusListsByGuid': {'LL00049_pid_genes_autosomal_do': mock.ANY, 'LL00005_retina_proteome': mock.ANY}, 'totalSampleCounts': {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3, 'WGS': 3}} if genome_version == '38' else {}, 'variantsById': {v['variantId']: { + 'discoveryTagFamilies': [], + 'excludedTagFamilies': [], **v, 'familyGuids': [], 'lookupFamilyGuids': v['familyGuids'] + v.get('liftedFamilyGuids', []), @@ -1390,25 +1583,33 @@ def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version } self.assertDictEqual(response.json(), expected_body) - parsed_cached_variants = [] - for v in (cached_variants or variants): - family_genotypes = defaultdict(list) - for individual_guid, gts in v['genotypes'].items(): - if not isinstance(gts, list): - gts = [gts] - for gt in gts: - family_guid = gt.get('familyGuid') or expected_individuals[individual_guid]['familyGuid'] - family_genotypes[family_guid].append({k: v for k, v in gt.items() if k != 'individualGuid'}) - parsed_cached_variants.append({ - **{k: v for k, v in v.items() if k not in {'familyGuids', 'genotypes'}}, - 'familyGenotypes': { - family_guid: sorted(gts, key=lambda x: (x['sampleType'] == 'WES', x.get('sampleId')), reverse=True) - for family_guid, gts in family_genotypes.items() - }, - }) + parsed_cached_variants = [ + self._cached_lookup_variant(v, expected_individuals) for v in (cached_variants or variants) + ] self.assert_cached_results(parsed_cached_variants, cache_key) return url + def _cached_lookup_variant(self, variant, expected_individuals=None): + family_genotypes = defaultdict(list) + for individual_guid, gts in variant['genotypes'].items(): + if not isinstance(gts, list): + gts = [gts] + for gt in gts: + family_guid = gt.get('familyGuid') or expected_individuals[individual_guid]['familyGuid'] + family_genotypes[family_guid].append({ + **{k: v for k, v in gt.items() if k != 'individualGuid'}, + 'metadata': self.INDIVIDUAL_METADATA.get(gt['individualGuid']), + }) + return { + 'discoveryFamilies': [], + 'excludedTagFamilies': [], + **{k: v for k, v in variant.items() if k not in {'familyGuids', 'genotypes'}}, + 'familyGenotypes': { + family_guid: sorted(gts, key=lambda x: (x['sampleType'] == 'WES', x.get('sampleId')), reverse=True) + for family_guid, gts in family_genotypes.items() + }, + } + def test_get_single_variant(self): url_template = (reverse(query_single_variant_handler, args=['variant_id']) + '?familyGuid={}').replace('variant_id', '{}') url = url_template.format('21-3343353-GAGA-G', 'F000001_1') @@ -1451,8 +1652,8 @@ def test_get_single_variant(self): self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json()['variantsById'], {'7-143270172-A-G': GRCH37_VARIANT}) - self.mock_redis.get.assert_not_called() - self.mock_redis.set.assert_not_called() + self.assertTrue(all(call.args[0].startswith('projects__') for call in self.mock_redis.get.mock_calls)) + self.assertTrue(all(call.args[0].startswith('projects__') for call in self.mock_redis.set.mock_calls)) def test_frequency_filter(self): sv_callset_filter = {'sv_callset': {'af': 0.05}} @@ -1837,6 +2038,16 @@ def test_secondary_annotations_filter(self): ], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], {}], ) + self._assert_expected_search( + [PROJECT_2_VARIANT1, VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4], MITO_VARIANT3], + inheritance_mode='recessive', project_families=MULTI_PROJECT_PROJECT_FAMILIES, pathogenicity=pathogenicity, + locus={'rawItems': 'chr1:1-100000000, chr14:1-100000000, chr16:1-100000000, chr17:1-100000000, M:1-100000000'}, + annotations=gcnv_annotations_2, annotations_secondary=gcnv_annotations_1, cached_variant_fields=[{}, {}, [ + {'selectedGeneId': 'ENSG00000277258'}, + {'selectedGeneId': 'ENSG00000277258'}, + ], {}, [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], {}], + ) + selected_transcript_annotations = {'other': ['non_coding_transcript_exon_variant']} self._assert_expected_search( [VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], GCNV_VARIANT3, MITO_VARIANT3], @@ -1936,7 +2147,7 @@ def test_in_silico_filter(self): ) self._assert_expected_search( - [VARIANT2, MULTI_FAMILY_VARIANT], in_silico={'gnomad_noncoding': 0.5, 'requireScore': True}, + [VARIANT2, MULTI_FAMILY_VARIANT], in_silico={'gnomad_noncoding': 0.5, 'vest': None, 'requireScore': True}, ) self._assert_expected_search( @@ -1954,6 +2165,9 @@ def test_in_silico_filter(self): self._assert_expected_search( [SV_VARIANT4], in_silico=sv_in_silico, project_families=SV_PROJECT_FAMILIES, ) + self._assert_expected_search( + [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3], in_silico={'strvctvre': 0.2}, project_families=SV_PROJECT_FAMILIES, + ) self._set_grch37_search() self._assert_expected_search([GRCH37_VARIANT], in_silico=main_in_silico, is_37=True) @@ -2008,7 +2222,7 @@ def test_sort(self): self._assert_expected_search( [MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, VARIANT4, MULTI_FAMILY_VARIANT, VARIANT1, VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, GCNV_VARIANT2, GCNV_VARIANT1], - sort='callset_af', + sort='seqr_ac', ) self._assert_expected_search( @@ -2078,7 +2292,7 @@ def test_sort(self): self._assert_expected_search( [MITO_VARIANT3, [VARIANT4, VARIANT3], VARIANT2], - sort='callset_af', inheritance_mode='recessive', **ALL_SNV_INDEL_PASS_FILTERS, cached_variant_fields=[ + sort='seqr_ac', inheritance_mode='recessive', **ALL_SNV_INDEL_PASS_FILTERS, cached_variant_fields=[ {}, [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], {}, ], project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) @@ -2138,7 +2352,7 @@ def test_multi_data_type_comp_het_sort(self): self._assert_expected_search( [[VARIANT4, VARIANT3], [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [GCNV_VARIANT3, GCNV_VARIANT4]], - sort='callset_af', inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, cached_variant_fields=[ + sort='seqr_ac', inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, cached_variant_fields=[ [{'selectedGeneId': 'ENSG00000097046'}, {'selectedGeneId': 'ENSG00000097046'}], [{'selectedGeneId': 'ENSG00000277258'}, {'selectedGeneId': 'ENSG00000277258'}], [{'selectedGeneId': 'ENSG00000275023'}, {'selectedGeneId': 'ENSG00000275023'}], @@ -2261,10 +2475,6 @@ def test_gene_variant_lookup(self): ) def test_search_context(self): - sv = SavedVariant.objects.get(guid='SV0000001_2103343353_r0390_100') - sv.saved_variant_json['genomeVersion'] = '38' - sv.save() - expected_results = [ VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, FAMILY_1_VARIANT, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, @@ -2408,22 +2618,8 @@ def test_search_context(self): '3': {'chrom': '1', 'end': 249055991, 'mimNumber': 600315, 'phenotypeDescription': '?Immunodeficiency 16', 'phenotypeInheritance': 'Autosomal recessive', 'phenotypeMimNumber': 615120, 'start': 249044482}, }) - # Test cross-project discovery for analyst users - self.set_cache('search_results__VRS0009876__xpos',[{ - 'key': 100, 'familyGuids': ['F000002_2'], 'xpos': 1248367227, 'genomeVersion': '38', 'sortedTranscriptConsequences': [], - }]) - response, _, _ = self._execute_search(search_hash=9876) - self.assertEqual(response.status_code, 200) - variants = response.json()['variantsById'] - self.assertEqual(len(variants), 1) - self.assertFalse('discoveryTags' in variants['1-248367227-TC-T']) - - self.login_analyst_user() - response, _, _ = self._execute_search(search_hash=9876) - self.assertEqual(response.status_code, 200) - variants = response.json()['variantsById'] - self.assertEqual(len(variants), 1) - self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], [{ + # Test cross-project discovery tags + discovery_tag = { 'savedVariant': { 'variantGuid': 'SV0000006_1248367227_r0003_tes', 'familyGuid': 'F000012_12', @@ -2438,8 +2634,26 @@ def test_search_context(self): 'metadata': None, 'lastModifiedDate': '2018-05-29T16:32:51.449Z', 'createdBy': None, - }]) - self.assertDictEqual(response.json()['familiesByGuid'], {'F000012_12': mock.ANY}) + } + self._add_sample_type_samples('WGS', active_individuals__family__guid='F000002_2') + self._assert_expected_search( + [{**DISCOVERY_VARIANT, 'noAccessDiscoveryFamilies': 0, 'discoveryTags': [discovery_tag, { + **discovery_tag, + 'savedVariant': { + 'variantGuid': 'SV0000006_1248367227_r0004_non', + 'familyGuid': 'F000014_14', + 'projectGuid': 'R0004_non_analyst_project', + }, + 'tagGuid': 'VT1726961_2103343353_r0005_tes', + }]}], locus={'rawVariantItems': '1-248367227-TC-T'}, additional_response={ + 'familiesByGuid': {'F000012_12': mock.ANY, 'F000014_14': mock.ANY}, + }) + + self.login_analyst_user() + self._assert_expected_search( + [{**DISCOVERY_VARIANT, 'noAccessDiscoveryFamilies': 1, 'discoveryTags': [discovery_tag]}], + locus={'rawVariantItems': '1-248367227-TC-T'}, additional_response={'familiesByGuid': {'F000012_12': mock.ANY}}, + ) def test_cached_query_variants(self): search_hash = 987 diff --git a/clickhouse_search/backend/table_models.py b/clickhouse_search/backend/table_models.py index c8cb45a684..819ca96972 100644 --- a/clickhouse_search/backend/table_models.py +++ b/clickhouse_search/backend/table_models.py @@ -81,13 +81,13 @@ def base_fields(cls): return [(field.db_column or field.name, field) for field in cls._meta.local_fields if field.name != 'key'] @classmethod - def dict_get_expression(cls, *expressions, field_names=None, force_tuple=False, **kwargs): + def dict_get_expression(cls, *expressions, key_template='%(expressions)s', field_names=None, force_tuple=False, **kwargs): base_fields = cls.base_fields() if field_names: base_fields = [f for f in base_fields if f[0] in field_names] output_field = base_fields[0][1] if len(base_fields) == 1 and not force_tuple else NamedTupleField(base_fields) dict_get_func = Func(*expressions, output_field=output_field) - dict_get_func.template = cls.dict_get_sql('%(expressions)s', [field_name for field_name, _ in base_fields], **kwargs) + dict_get_func.template = cls.dict_get_sql(key_template, [field_name for field_name, _ in base_fields], **kwargs) if force_tuple and len(base_fields) == 1: dict_get_func.template = f'tuple({dict_get_func.template})' return dict_get_func diff --git a/clickhouse_search/fixtures/clickhouse_discovery_variants.json b/clickhouse_search/fixtures/clickhouse_discovery_variants.json new file mode 100644 index 0000000000..83ff2e903d --- /dev/null +++ b/clickhouse_search/fixtures/clickhouse_discovery_variants.json @@ -0,0 +1,88 @@ +[{ + "model": "clickhouse_search.entriessnvindel", + "pk": 100, + "fields": { + "key": 100, + "project_guid": "R0001_1kg", + "family_guid": "F000002_2", + "sample_type": "WGS", + "xpos": 1248367227, + "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": true, + "geneId_ids": [6, 48], + "filters": [], + "sign": 1, + "calls": [ + ["HG00733", 1, 99, 0.6530612111091614, 49], + ["HG00731", 0, 48, 1, 16] + ] + } +}, { + "model": "clickhouse_search.keylookupsnvindel", + "pk": 100, + "fields": { + "key": 100, + "variant_id": "1-248367227-TC-T" + } +}, { + "model": "clickhouse_search.variantssnvindel", + "pk": 100, + "fields": { + "key": 100, + "sorted_motif_feature_consequences": [], + "sorted_regulatory_feature_consequences": [], + "sorted_transcript_consequences": [] + } +},{ + "model": "clickhouse_search.variantdetailssnvindel", + "pk": 100, + "fields": { + "key": 100, + "variant_id": "1-248367227-TC-T", + "rsid": null, + "caid": "CA1501729", + "lifted_over_chrom": "1", + "lifted_over_pos": null, + "transcripts": [], + "sorted_motif_feature_consequences": [], + "sorted_regulatory_feature_consequences": [] + } +}, { + "model": "clickhouse_search.entriessnvindel", + "pk": 100, + "fields": { + "key": 100, + "project_guid": "R0004_non_analyst_project", + "family_guid": "F000014_14", + "sample_type": "WGS", + "xpos": 1248367227, + "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], + "filters": [], + "sign": 1, + "calls": [ + ["NA21234", 2, 99, 0, 49] + ] + } +}, { + "model": "clickhouse_search.entriessnvindel", + "pk": 100, + "fields": { + "key": 100, + "project_guid": "R0003_test", + "family_guid": "F000012_12", + "sample_type": "WGS", + "xpos": 1248367227, + "is_gnomad_gt_5_percent": false, + "is_annotated_in_any_gene": false, + "geneId_ids": [], + "filters": [], + "sign": 1, + "calls": [ + ["NA20889", 1, 99, 0.0, 71], + ["NA20888", 1, 99, 0.5555556, 9] + ] + } +} +] \ No newline at end of file diff --git a/clickhouse_search/fixtures/clickhouse_saved_variants.json b/clickhouse_search/fixtures/clickhouse_saved_variants.json index 69197c9a56..c4a1113b6c 100644 --- a/clickhouse_search/fixtures/clickhouse_saved_variants.json +++ b/clickhouse_search/fixtures/clickhouse_saved_variants.json @@ -172,45 +172,6 @@ "sift": null, "vest": null } -}, { - "model": "clickhouse_search.keylookupsnvindel", - "pk": 100, - "fields": { - "key": 100, - "variant_id": "1-248367227-TC-T" - } -}, { - "model": "clickhouse_search.variantdetailssnvindel", - "pk": 100, - "fields": { - "key": 100, - "variant_id": "1-248367227-TC-T", - "rsid": null, - "caid": "CA1501729", - "lifted_over_chrom": "1", - "lifted_over_pos": null, - "transcripts": [], - "sorted_motif_feature_consequences": [], - "sorted_regulatory_feature_consequences": [] - } -}, { - "model": "clickhouse_search.entriessnvindel", - "pk": 100, - "fields": { - "key": 100, - "project_guid": "R0004_non_analyst_project", - "family_guid": "F000014_14", - "sample_type": "WGS", - "xpos": 1248367227, - "is_gnomad_gt_5_percent": false, - "is_annotated_in_any_gene": false, - "geneId_ids": [], - "filters": [], - "sign": 1, - "calls": [ - ["NA21234", 2, 99, 0, 49] - ] - } }, { "model": "clickhouse_search.keylookupsv", "pk": 101, diff --git a/clickhouse_search/management/commands/register_caids.py b/clickhouse_search/management/commands/register_caids.py index dd0455bc93..72d13929e6 100644 --- a/clickhouse_search/management/commands/register_caids.py +++ b/clickhouse_search/management/commands/register_caids.py @@ -108,8 +108,8 @@ class AlleleRegistryError: def from_api_response(cls, response: dict): return cls( error_type=response["errorType"], - description=response["description"], - message=response["message"], + description=response.get("description"), + message=response.get("message"), input_line=response.get("inputLine"), ) @@ -154,8 +154,10 @@ def handle_api_response( genome_version: Literal[GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38], response: requests.Response, ) -> dict[str, str]: + if not response.ok: + raise HTTPError(f"Unexpected AR response code: {response.status_code}") response_json = response.json() - if not response.ok or "errorType" in response_json: + if "errorType" in response_json: error = AlleleRegistryError.from_api_response(response_json) logger.error(error) raise HTTPError(error.message) diff --git a/clickhouse_search/management/commands/reload_clinvar_all_variants.py b/clickhouse_search/management/commands/reload_clinvar_all_variants.py index 50c2ac478e..de9793c961 100644 --- a/clickhouse_search/management/commands/reload_clinvar_all_variants.py +++ b/clickhouse_search/management/commands/reload_clinvar_all_variants.py @@ -20,27 +20,19 @@ logger = logging.getLogger(__name__) -def replace_underscores_with_spaces(value: Union[str, list[str]]) -> Union[str, list[str]]: - if isinstance(value, str): - return value.replace('_', ' ') - elif isinstance(value, list): - return [s.replace('_', ' ') for s in value] - raise TypeError("Expected str or list[str]") +def replace_underscores_with_spaces(value: list[str]) -> list[str]: + return [s.replace('_', ' ') for s in value] -def replace_spaces_with_underscores(value: Union[str, list[str], list[tuple[str, int]]]) -> Union[str, list[str]]: - if isinstance(value, str): - return value.replace(' ', '_') - elif isinstance(value, list): - if len(value) > 0 and isinstance(value[0], tuple): - return [(t[0].replace(' ', '_'), t[1]) for t in value] - return [s.replace(' ', '_') for s in value] - raise TypeError("Expected str or list[str]") +def replace_spaces_with_underscores(value: Union[list[str], list[tuple[str, int]]]) -> list[str]: + if len(value) > 0 and isinstance(value[0], tuple): + return [(t[0].replace(' ', '_'), t[1]) for t in value] + return [s.replace(' ', '_') for s in value] BATCH_SIZE = 1000 CLINVAR_ASSERTIONS = replace_underscores_with_spaces(ClinvarAllVariantsSnvIndel.CLINVAR_ASSERTIONS) -CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY = replace_underscores_with_spaces(ClinvarAllVariantsSnvIndel.CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY) +CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY = replace_underscores_with_spaces([ClinvarAllVariantsSnvIndel.CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY])[0] CLINVAR_CONFLICTING_DATA_FROM_SUBMITTERS = 'conflicting data from submitters' -CLINVAR_DEFAULT_PATHOGENICITY = replace_underscores_with_spaces(ClinvarAllVariantsSnvIndel.CLINVAR_DEFAULT_PATHOGENICITY) +CLINVAR_DEFAULT_PATHOGENICITY = replace_underscores_with_spaces([ClinvarAllVariantsSnvIndel.CLINVAR_DEFAULT_PATHOGENICITY])[0] CLINVAR_PATHOGENICITIES = replace_underscores_with_spaces(ClinvarAllVariantsSnvIndel.CLINVAR_PATHOGENICITIES) CLINVAR_GOLD_STARS_LOOKUP = { 'no classification for the single variant': 0, @@ -228,7 +220,7 @@ def extract_variant_info(elem: xml.etree.ElementTree.Element, new_version: str, props = { 'version': new_version, 'allele_id': allele_id, - 'pathogenicity': replace_spaces_with_underscores(pathogenicity), + 'pathogenicity': replace_spaces_with_underscores([pathogenicity])[0], 'assertions': replace_spaces_with_underscores(assertions), 'conflicting_pathogenicities': replace_spaces_with_underscores(conflicting_pathogenicities), 'gold_stars': gold_stars, @@ -261,7 +253,7 @@ def parse_clinvar_file(gzipped_file, existing_version_obj, model_to_batch, unenu if existing_version_obj: if existing_version_obj.version == new_version: logger.info(f'Clinvar ClickHouse tables already successfully updated to {new_version}, gracefully exiting.') - return new_version + return None logger.info( f'Updating Clinvar ClickHouse tables to {new_version} from {existing_version_obj and existing_version_obj.version}.') # Drop any currently existing variants in the table that may exist due to a # previously failed partial run. Note that we validate that the Postgresql existing version @@ -306,6 +298,9 @@ def handle(self, *args, **options): with gzip.open(tmpfile.name, 'rb') as gzipped_file: new_version = parse_clinvar_file(gzipped_file, existing_version_obj, model_to_batch, unenumerated_value_alerts) + if not new_version: + return + for model, batch in model_to_batch.items(): if batch: model.objects.using('clickhouse_write').bulk_create(batch) diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py index 0928b6d728..6b5eedabcd 100644 --- a/clickhouse_search/management/commands/set_saved_variant_key.py +++ b/clickhouse_search/management/commands/set_saved_variant_key.py @@ -19,7 +19,6 @@ class Command(BaseCommand): def handle(self, *args, **options): variant_ids = SavedVariant.objects.filter( key__isnull=True, family__project__genome_version=GENOME_VERSION_GRCh38, - saved_variant_json__populations__isnull=False, # Omit manual variants ).values_list('variant_id', flat=True).distinct() ids_by_dataset_type = { Dataset.DATASET_TYPE_VARIANT_CALLS: [], Dataset.DATASET_TYPE_MITO_CALLS: [], Dataset.DATASET_TYPE_SV_CALLS: [], @@ -59,11 +58,6 @@ def handle(self, *args, **options): if no_keys_37: self._resolve_missing_variants(no_keys_37, GENOME_VERSION_GRCh37) - num_updated = SavedVariant.objects.filter(key__isnull=False).exclude(saved_variant_json={}).update( - saved_variant_json={}, - ) - logger.info(f'Cleared saved json for {num_updated} variants with keys') - logger.info('Done') @staticmethod @@ -98,32 +92,32 @@ def _set_variant_keys(variants_ids, dataset_type, genome_version=GENOME_VERSION_ return no_key @classmethod - def _query_missing_variants(cls, variant_ids, variant_fields, genome_version=GENOME_VERSION_GRCh38): + def _query_missing_variants(cls, variant_ids, variant_fields=None, genome_version=GENOME_VERSION_GRCh38): missing_variants = SavedVariant.objects.filter( variant_id__in=variant_ids, family__project__genome_version=genome_version, ) num_missing = missing_variants.count() missing_with_data_qs = missing_variants.filter(family__individual__active_datasets__isnull=False).distinct() missing_with_search_data = missing_with_data_qs.values( - 'variant_id', *variant_fields, + 'variant_id', *(variant_fields or []), ).annotate(family_ids=ArrayAgg('family__family_id', distinct=True)).order_by('variant_id') return missing_with_search_data, num_missing @classmethod def _resolve_missing_variants(cls, variant_ids, genome_version): missing_with_search_data, num_missing = cls._query_missing_variants( - variant_ids, ['saved_variant_json__populations__seqr__ac'], genome_version, + variant_ids, genome_version=genome_version, ) num_data= len(missing_with_search_data) - in_backend = [ - f"{var['variant_id']} - {'; '.join(var['family_ids'])}" - for var in missing_with_search_data if var['saved_variant_json__populations__seqr__ac'] - ] + logger.info( - f'{num_missing} variants have no key, {num_missing - num_data} of which have no search data, {num_data - len(in_backend)} of which are absent from the hail backend.' + f'{num_missing} variants have no key, {num_missing - num_data} of which have no search data.' ) - if in_backend: - logger.info(f'{len(in_backend)} remaining variants: {", ".join(in_backend)}') + if missing_with_search_data: + summary = [ + f"{var['variant_id']} - {'; '.join(var['family_ids'])}" for var in missing_with_search_data + ] + logger.info(f'{len(missing_with_search_data)} remaining variants: {", ".join(summary)}') @classmethod def _resolve_reloaded_svs(cls, variant_ids): diff --git a/clickhouse_search/management/tests/register_caids_tests.py b/clickhouse_search/management/tests/register_caids_tests.py index 9985799c50..d0c27f257f 100644 --- a/clickhouse_search/management/tests/register_caids_tests.py +++ b/clickhouse_search/management/tests/register_caids_tests.py @@ -50,6 +50,26 @@ }, ], }, + { + '@id': 'http://reg.genome.network/allele/CA16716504', + 'genomicAlleles': [ + { + 'chromosome': '1', + 'coordinates': [ + { + 'allele': 'C', + 'end': 10131, + 'referenceAllele': '', # ref allele is '' + 'start': 10131, + }, + ], + 'referenceGenome': 'GRCh38', + }, + ], + 'externalRecords': { + 'gnomAD_4': [{'id': '1-91511686-91511734'}], # has invalid gnomad ID + }, + }, { '@id': 'http://reg.genome.network/allele/CA997563845', 'genomicAlleles': [ @@ -135,6 +155,21 @@ def test_bad_responses(self, mock_safe_post_to_slack, mock_logger): call_command("register_caids", batch_size=5) mock_safe_post_to_slack.assert_not_called() + responses.add( + responses.PUT, + "https://reg.genome.network/alleles", + match=[ + responses.matchers.query_param_matcher({ + "file": "vcf", + "fields": "none @id genomicAlleles externalRecords.gnomAD_4.id", + }, strict_match=False), + ], + status=400, + ) + with self.assertRaisesMessage(CommandError, 'Failed in 38/ClingenAlleleRegistry curr_key: 3'): + call_command("register_caids", batch_size=5) + mock_safe_post_to_slack.assert_not_called() + responses.reset() mock_safe_post_to_slack.reset_mock() mock_logger.reset_mock() @@ -224,7 +259,7 @@ def test_register_caids(self, mock_safe_post_to_slack, mock_logger): dv_38 = DataVersions.objects.get(data_model_name='38/ClingenAlleleRegistry') self.assertEqual(dv_38.version, '10') mock_logger.info.assert_called_with( - "2 registered variant(s) cannot be mapped back to ours. \n" + "3 registered variant(s) cannot be mapped back to ours. \n" "First unmappable variant:\n{'@id': 'http://reg.genome.network/allele/CA16716503', 'genomicAlleles': [{'chromosome': '1', 'coordinates': [{'allele': 'C', 'end': 10131, 'referenceAllele': '', 'start': 10131}], 'referenceGenome': 'GRCh38'}]}" ) mock_logger.warning.assert_called_with( diff --git a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py index c22bff5763..b7a18267d5 100644 --- a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py +++ b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py @@ -103,6 +103,8 @@ def test_new_version_already_exists(self, mock_logger, mock_safe_post_to_slack): responses.add(responses.GET, WEEKLY_XML_RELEASE, status=200, body=gzip.compress(WEEKLY_XML_RELEASE_DATA.encode()), stream=True) call_command('reload_clinvar_all_variants') mock_logger.assert_called_with('Clinvar ClickHouse tables already successfully updated to 2025-06-30, gracefully exiting.') + mock_safe_post_to_slack.assert_not_called() + self.assertEqual(DataVersions.objects.get(data_model_name='Clinvar').version, '2025-06-30') @responses.activate def test_parse_variants_all_types(self, mock_logger, mock_safe_post_to_slack): diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py index 9b3d07f479..daf46291fa 100644 --- a/clickhouse_search/management/tests/set_saved_variant_key_tests.py +++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py @@ -11,7 +11,7 @@ class SetSavedVariantKeyTest(AnvilAuthenticationTestCase): - fixtures = ['users', '1kg_project', 'report_variants', 'clickhouse_saved_variants'] + fixtures = ['users', '1kg_project', 'report_variants', 'clickhouse_discovery_variants', 'clickhouse_saved_variants'] MOCK_GCNV_DATA = MOCK_GCNV_DATA @@ -22,13 +22,6 @@ def setUpTestData(cls): dataset.dataset_type = 'SV' dataset.save() dataset.active_individuals.set({17}) - for sv in SavedVariant.objects.filter(key__isnull=False): - sv.saved_variant_json = { - 'genotypes': sv.genotypes, 'populations': {'gnomad': {'af': 0.01}}, - } - if sv.guid == 'SV0000009_25000014783_r0004_no': - sv.saved_variant_json['populations']['seqr'] = {'af': 0.019480518996715546, 'ac': 3, 'an': 154} - sv.save() SavedVariant.objects.update(key=None) SavedVariant.objects.filter(guid='SV0000007_prefix_19107_DEL_r00').update(variant_id='prefix_19107_DEL') @@ -62,25 +55,25 @@ def test_command(self, mock_subprocess): ('Updated batch of 1', None), ('Updated keys for 1 SNV_INDEL (GRCh37) variants', None), ('No key found for 6 variants', None), - ('6 variants have no key, 0 of which have no search data, 6 of which are absent from the hail backend.', None), - ('Cleared saved json for 6 variants with keys', None), + ('6 variants have no key, 0 of which have no search data.', None), + ('6 remaining variants: 1-1562437-G-CA - 1, 1-46859832-G-A - 1, 19-1912632-G-C - 2, 19-1912633-G-T - 2, 19-1912634-C-T - 2, 21-3343353-GAGA-G - 1', None), ('Done', None), ]) - saved_variants = list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id', 'saved_variant_json')) + saved_variants = list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id')) expected_saved_variants = [ - {'guid': 'SV0000001_2103343353_r0390_100', 'key': None, 'variant_id': '21-3343353-GAGA-G', 'saved_variant_json': mock.ANY}, - {'guid': 'SV0000002_1248367227_r0390_100', 'key': 100, 'variant_id': '1-248367227-TC-T', 'saved_variant_json': {}}, - {'guid': 'SV0000006_1248367227_r0003_tes', 'key': 100, 'variant_id': '1-248367227-TC-T', 'saved_variant_json': {}}, - {'guid': 'SV0000006_1248367227_r0004_non', 'key': 100, 'variant_id': '1-248367227-TC-T', 'saved_variant_json': {}}, - {'guid': 'SV0000007_prefix_19107_DEL_r00', 'key': 111, 'variant_id': 'prefix_19107_DEL', 'saved_variant_json': {}}, - {'guid': 'SV0000009_25000014783_r0004_no', 'key': 100, 'variant_id': 'M-14783-T-C', 'saved_variant_json': {}}, - {'guid': 'SV0000013_prefix_19107_DEL_r00', 'key': 101, 'variant_id': 'suffix_19107_DEL_013746', 'saved_variant_json': {}}, - {'guid': 'SV0027166_191912634_r0384_rare', 'key': None, 'variant_id': '19-1912634-C-T', 'saved_variant_json': mock.ANY}, - {'guid': 'SV0027167_191912633_r0384_rare', 'key': None, 'variant_id': '19-1912633-G-T', 'saved_variant_json': mock.ANY}, - {'guid': 'SV0027168_191912632_r0384_rare', 'key': None, 'variant_id': '19-1912632-G-C', 'saved_variant_json': mock.ANY}, - {'guid': 'SV0059956_11560662_f019313_1', 'key': None, 'variant_id': '1-46859832-G-A', 'saved_variant_json': mock.ANY}, - {'guid': 'SV0059957_11562437_f019313_1', 'key': None, 'variant_id': '1-1562437-G-CA', 'saved_variant_json': mock.ANY}, + {'guid': 'SV0000001_2103343353_r0390_100', 'key': None, 'variant_id': '21-3343353-GAGA-G'}, + {'guid': 'SV0000002_1248367227_r0390_100', 'key': 100, 'variant_id': '1-248367227-TC-T'}, + {'guid': 'SV0000006_1248367227_r0003_tes', 'key': 100, 'variant_id': '1-248367227-TC-T'}, + {'guid': 'SV0000006_1248367227_r0004_non', 'key': 100, 'variant_id': '1-248367227-TC-T'}, + {'guid': 'SV0000007_prefix_19107_DEL_r00', 'key': 111, 'variant_id': 'prefix_19107_DEL'}, + {'guid': 'SV0000009_25000014783_r0004_no', 'key': 100, 'variant_id': 'M-14783-T-C'}, + {'guid': 'SV0000013_prefix_19107_DEL_r00', 'key': 101, 'variant_id': 'suffix_19107_DEL_013746'}, + {'guid': 'SV0027166_191912634_r0384_rare', 'key': None, 'variant_id': '19-1912634-C-T'}, + {'guid': 'SV0027167_191912633_r0384_rare', 'key': None, 'variant_id': '19-1912633-G-T'}, + {'guid': 'SV0027168_191912632_r0384_rare', 'key': None, 'variant_id': '19-1912632-G-C'}, + {'guid': 'SV0059956_11560662_f019313_1', 'key': None, 'variant_id': '1-46859832-G-A'}, + {'guid': 'SV0059957_11562437_f019313_1', 'key': None, 'variant_id': '1-1562437-G-CA'}, ] self.assertListEqual(saved_variants, expected_saved_variants) @@ -90,12 +83,12 @@ def test_command(self, mock_subprocess): self.assert_json_logs(user=None, expected=[ ('Finding keys for 6 SNV_INDEL (GRCh37) variant ids', None), ('Found 0 keys', None), - ('6 variants have no key, 0 of which have no search data, 6 of which are absent from the hail backend.', None), - ('Cleared saved json for 0 variants with keys', None), + ('6 variants have no key, 0 of which have no search data.', None), + ('6 remaining variants: 1-1562437-G-CA - 1, 1-46859832-G-A - 1, 19-1912632-G-C - 2, 19-1912633-G-T - 2, 19-1912634-C-T - 2, 21-3343353-GAGA-G - 1', None), ('Done', None), ]) self.assertListEqual( - list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id', 'saved_variant_json')), + list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id')), expected_saved_variants, ) @@ -115,8 +108,8 @@ def test_command(self, mock_subprocess): ('Found 0 keys', None), ('Finding keys for 2 SNV_INDEL (GRCh38) variant ids', None), ('Found 0 keys', None), - ('3 variants have no key, 1 of which have no search data, 1 of which are absent from the hail backend.', None), - ('1 remaining variants: M-14783-T-C - fam14', None), + ('3 variants have no key, 1 of which have no search data.', None), + ('2 remaining variants: 1-248367227-TC-T - 12; fam14, M-14783-T-C - fam14', None), ('Finding keys for 2 SV_WGS (GRCh38) variant ids', None), ('Found 0 keys', None), ('Finding keys for 2 SV_WES (GRCh38) variant ids', None), @@ -126,8 +119,8 @@ def test_command(self, mock_subprocess): ('1 remaining SV WGS variants suffix_19107_DEL_013746 - fam14', None), ('Finding keys for 7 SNV_INDEL (GRCh37) variant ids', None), ('Found 0 keys', None), - ('7 variants have no key, 0 of which have no search data, 7 of which are absent from the hail backend.', None), - ('Cleared saved json for 0 variants with keys', None), + ('7 variants have no key, 0 of which have no search data.', None), + ('7 remaining variants: 1-1562437-G-CA - 1, 1-248367227-TC-T - 2, 1-46859832-G-A - 1, 19-1912632-G-C - 2, 19-1912633-G-T - 2, 19-1912634-C-T - 2, 21-3343353-GAGA-G - 1', None), ('Done', None), ]) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index b19f8f2250..a1f64006d7 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -11,7 +11,7 @@ ArrayIntersect, ArrayJoin, ArrayMap, ArraySort, ArraySymmetricDifference, CrossJoin, GroupArray, GroupArrayArray, \ GroupArrayIntersect, If, MapLookup, NullIf, Plus, SubqueryJoin, SubqueryTable, Tuple, TupleConcat, Untuple, \ IntDiv, Modulo, SplitByString, ArrayIndex, Multiply, IndexOf -from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict +from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict, DiscoveryVariantDict from clickhouse_search.constants import INHERITANCE_FILTERS, ANY_AFFECTED, AFFECTED, UNAFFECTED, MALE_SEXES, \ X_LINKED_RECESSIVE, REF_REF, REF_ALT, ALT_ALT, HAS_ALT, HAS_REF, SPLICE_AI_FIELD, SCREEN_KEY, UTR_ANNOTATOR_KEY, \ EXTENDED_SPLICE_KEY, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY, CLINVAR_KEY, HGMD_KEY, NEW_SV_FIELD, \ @@ -197,9 +197,9 @@ def annotation_values(self): if field.db_column and field.name != field.db_column and field.db_column } - def result_values(self, additional_fields=None, **kwargs): + def result_values(self, additional_fields=None, additional_values=None, **kwargs): fields = [*self.annotation_fields] + (additional_fields or []) - values = {**self.annotation_values} + values = {**self.annotation_values, **(additional_values or {})} values.update(self.conditional_selects(self, **kwargs)) override_model_annotations = set(values).intersection(fields) @@ -303,11 +303,6 @@ def entry_model(self): def gt_stats_dict_rel(self): return getattr(self.entry_model, 'gt_stats', None) - @property - def genome_version(self): - return self.model.ANNOTATION_CONSTANTS['genomeVersion'] - - def subquery_join(self, subquery, join_key='key'): join_field = next(field for field in subquery.model._meta.fields if field.name == join_key) @@ -406,13 +401,6 @@ def _comp_het_conditional_fields(self, query, prefix=''): if field in query.query.annotations } - @property - def populations(self): - return { - population: {subfield for subfield, _ in field.base_fields} - for population, field in self.model.POPULATION_FIELDS - } - def _filter_frequency(self, results, **kwargs): return results @@ -549,6 +537,9 @@ def annotation_values(self): if self.has_annotation('mitomapPathogenic'): annotations['mitomapPathogenic'] = F('mitomapPathogenic') + if not self.variant_detail_field: + annotations['discoveryFamilies'] = DiscoveryVariantDict.dict_get_expression('key', dataset_type='MITO') + return annotations @staticmethod @@ -764,6 +755,7 @@ class SvVariantsQuerySet(BaseVariantsQuerySet): def annotation_values(self): annotations = super().annotation_values annotations['transcripts'] = annotations.pop(self.model.sorted_gene_consequences.field.db_column) + annotations['discoveryFamilies'] = DiscoveryVariantDict.dict_get_expression('key', dataset_type=f'SV_{self.entry_model.SAMPLE_TYPE}') return annotations @staticmethod @@ -875,16 +867,16 @@ def _parse_in_silico_qs(self, results, in_silico, require_score, in_silico_qs, i return super()._parse_in_silico_qs(results, in_silico, require_score, in_silico_qs, in_silico_missing_qs) - def _filter_frequency(self, results, freqs=None, pathogenicity=None, **kwargs): + def _filter_frequency(self, results, freqs=None, **kwargs): for population, pop_filter in (freqs or {}).items(): - pop_subfields = self.populations.get(population) - if not pop_subfields: + pop_field = dict(self.model.POPULATION_FIELDS).get(population) + if not pop_field: continue if pop_filter.get('af') is not None and pop_filter['af'] < 1: results = results.filter(**{f'populations__{population}__af__lte': pop_filter['af']}) elif pop_filter.get('ac') is not None: - if 'ac' in pop_subfields: + if any(subfield == 'ac' for subfield, _ in pop_field.base_fields): ac_field = f'populations__{population}__ac' else: ac_field = f'ac_{population}' @@ -892,9 +884,6 @@ def _filter_frequency(self, results, freqs=None, pathogenicity=None, **kwargs): results = results.filter(**{f'{ac_field}__lte': pop_filter['ac']}) - if pop_filter.get('hh') is not None: - results = results.filter(**{f'populations__{population}__hom__lte': pop_filter['hh']}) - return results @@ -921,6 +910,7 @@ def annotation_values(self): annotations = { **super().annotation_values, **self.split_variant_id_annotations(), + 'discoveryFamilies': DiscoveryVariantDict.dict_get_expression('key', dataset_type='SNV_INDEL'), } if self.has_annotation('hgmd_join'): annotations.update({ @@ -1015,10 +1005,6 @@ def genotype_fields(self): def gt_stats_dict_rel(self): return getattr(self.model, 'gt_stats', None) - @property - def genome_version(self): - return self.annotations_model.ANNOTATION_CONSTANTS['genomeVersion'] - @property def filtered_chrom(self): return self.annotations_model.ANNOTATION_CONSTANTS.get('chrom') @@ -1055,7 +1041,7 @@ def _join_annotations(self, entries): def result_values(self, *args, sample_data=None, **kwargs): entries = self._join_annotations(self) - return self._search_call_data(entries, sample_data) + return self._search_call_data(entries, sample_data, **kwargs) def _filter_project_families(self, entries, sample_data): project_guids = sample_data['project_guids'] @@ -1312,7 +1298,7 @@ def annotation_fields(cls, entries): fields.append('seqrPop') return fields - def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, multi_sample_type_families=None, skip_entry_fields=False, override_annotations=None, **kwargs): + def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, multi_sample_type_families=None, skip_entry_fields=False, override_annotations=None, additional_expressions=None, **kwargs): if annotate_hom_alts: entries = entries.annotate(has_hom_alt=Q(calls__array_exists={'gt': (2,)})) @@ -1323,7 +1309,7 @@ def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, mu if skip_entry_fields: entries = entries.annotate(numFamilies=Count('family_guid')) else: - gt_field, gt_expression = self.genotype_expression(sample_data) + gt_field, gt_expression = self.genotype_expression(sample_data, additional_expressions) entries = entries.annotate( familyGuids=ArraySort(ArrayDistinct(GroupArray('family_guid'))), **{gt_field: GroupArrayArray(gt_expression)}, @@ -1363,7 +1349,7 @@ def _annotate_calls(self, entries, sample_data=None, annotate_hom_alts=False, mu return entries - def genotype_expression(self, sample_data=None): + def genotype_expression(self, sample_data=None, additional_expressions=None): family_samples = defaultdict(list) samples = (sample_data or {}).get('samples') or [] for s in samples: @@ -1378,6 +1364,9 @@ def genotype_expression(self, sample_data=None): genotype_expressions.insert(0, f"map({', '.join(sample_map)})[family_guid][x.sampleId]") output_base_fields.insert(0, ('individualGuid', models.StringField())) output_field_kwargs = {'group_by_key': 'individualGuid', 'flatten_groups': True} + if additional_expressions: + genotype_expressions += list(additional_expressions.keys()) + output_base_fields += list(additional_expressions.values()) return 'genotypes' if samples else 'familyGenotypes', ArrayFilter( ArrayMap( 'calls', @@ -1435,7 +1424,7 @@ def _family_passes_expression(pass_field): mapped_expression='x.1', output_field=models.ArrayField(models.StringField()), ) - def filter_locus(self, exclude_intervals=None, require_gene_filter=False, intervals=None, genes=None, variant_ids=None, inheritance_mode=None, **kwargs): + def filter_locus(self, exclude_intervals=None, intervals=None, genes=None, variant_ids=None, inheritance_mode=None, **kwargs): entries = self if variant_ids: @@ -1452,7 +1441,7 @@ def filter_locus(self, exclude_intervals=None, require_gene_filter=False, interv raise InvalidDatasetTypeException if genes or intervals: - entries = entries.filter(self._filter_locations_q(intervals, genes, require_gene_filter)) + entries = entries.filter(self._filter_locations_q(intervals, genes)) elif exclude_intervals: entries = entries.exclude(self._filter_locations_q(exclude_intervals)) @@ -1466,13 +1455,13 @@ def _parse_variant_ids(raw_variant_items): parsed_variant_ids[variant_id] = parse_variant_id(variant_id) return parsed_variant_ids - def _filter_locations_q(self, intervals, genes=None, require_gene_filter=False): + def _filter_locations_q(self, intervals, genes=None): locus_q = None if genes: should_filter_interval = self._can_filter_gene_interval(genes) if should_filter_interval: intervals = list((genes or {}).values()) + (intervals or []) - if require_gene_filter or (not should_filter_interval): + if not should_filter_interval: locus_q = Q(geneId_ids__bitmap_has_any=[gene['id'] for gene in genes.values()]) if intervals: @@ -1481,8 +1470,6 @@ def _filter_locations_q(self, intervals, genes=None, require_gene_filter=False): interval_q |= self._interval_query(**interval) if locus_q is None: locus_q = interval_q - elif require_gene_filter: - locus_q &= interval_q else: locus_q |= interval_q @@ -1493,7 +1480,7 @@ def _can_filter_gene_interval(self, genes): def search_padded_interval(self, chrom, pos, padding): interval_q = self._interval_query(chrom, start=max(pos - padding, MIN_POS), end=min(pos + padding, MAX_POS)) - return self.filter(interval_q).result_values() + return self.filter(interval_q) @staticmethod def _interval_query(chrom, start, end, **kwargs): diff --git a/clickhouse_search/migrations/0040_gnomadnoncodingconstraintdict.py b/clickhouse_search/migrations/0040_gnomadnoncodingconstraintdict.py index 35376dfd70..274fb30bf1 100644 --- a/clickhouse_search/migrations/0040_gnomadnoncodingconstraintdict.py +++ b/clickhouse_search/migrations/0040_gnomadnoncodingconstraintdict.py @@ -4620,4 +4620,9 @@ class Migration(migrations.Migration): reference_dataset="promoterAI", ), ), + migrations.RunPython( + conditionally_refresh_reference_dataset( + reference_dataset="eigen", + ), + ), ] diff --git a/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py b/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py new file mode 100644 index 0000000000..66f666513a --- /dev/null +++ b/clickhouse_search/migrations/0043_discoveryvariantdict_excludedvariantdict_and_more.py @@ -0,0 +1,79 @@ +# Generated by Django 4.2.27 on 2026-05-14 21:48 + +import clickhouse_backend.models +from django.db import migrations +import django.db.models.manager + + +class Migration(migrations.Migration): + + dependencies = [ + ('clickhouse_search', '0042_delete_annotationsdiskgcnv_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='DiscoveryVariantDict', + fields=[ + ('key', clickhouse_backend.models.UInt32Field(primary_key=True, serialize=False)), + ('dataset_type', clickhouse_backend.models.StringField()), + ('family_guids', + clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField())), + ], + options={ + 'db_table': 'seqrdb_discovery_variant_dict', + 'engine': clickhouse_backend.models.MergeTree(primary_key=('key', 'dataset_type')), + 'layout': 'COMPLEX_KEY_HASHED()', + 'postgres_query': "SELECT sv.key as key, sv.dataset_type as dataset_type, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.category = ''CMG Discovery Tags'') GROUP BY sv.key, sv.dataset_type", + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='ExcludedVariantDict', + fields=[ + ('key', clickhouse_backend.models.UInt32Field(primary_key=True, serialize=False)), + ('dataset_type', clickhouse_backend.models.StringField()), + ('family_guids', + clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField())), + ], + options={ + 'db_table': 'seqrdb_excluded_variant_dict', + 'engine': clickhouse_backend.models.MergeTree(primary_key=('key', 'dataset_type')), + 'layout': 'COMPLEX_KEY_HASHED()', + 'postgres_query': "SELECT sv.key as key, sv.dataset_type as dataset_type, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.name = ''Excluded'') GROUP BY sv.key, sv.dataset_type", + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='IndividualMetadataDict', + fields=[ + ('family_guid', clickhouse_backend.models.StringField(primary_key=True, serialize=False)), + ('sampleId', clickhouse_backend.models.StringField()), + ('restrict_sharing', clickhouse_backend.models.BoolField()), + ('features', clickhouse_backend.models.StringField()), + ('omim_id', clickhouse_backend.models.UInt32Field()), + ('mondo_id', clickhouse_backend.models.StringField()), + ('is_solved', clickhouse_backend.models.BoolField()), + ('vlm_contact_email', clickhouse_backend.models.StringField()), + ], + options={ + 'db_table': 'seqrdb_individual_metadata_dict', + 'engine': clickhouse_backend.models.MergeTree(primary_key=('family_guid', 'sampleId')), + 'layout': 'COMPLEX_KEY_HASHED()', + 'postgres_query': "select f.guid as family_guid, i.individual_id as sampleId, p.restrict_sharing as restrict_sharing, i.features as features, f.post_discovery_omim_numbers[1] as omim_id, f.post_discovery_mondo_id as mondo_id, f.analysis_status in (''S'', ''S_kgfp'', ''S_kgdp'', ''S_ng'', ''ES'') as is_solved, p.vlm_contact_email as vlm_contact_email FROM seqr_individual i INNER JOIN seqr_family f ON i.family_id = f.id INNER JOIN seqr_project p ON f.project_id = p.id", + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.RunSQL('SYSTEM RELOAD DICTIONARY "seqrdb_discovery_variant_dict"'), + migrations.RunSQL('SYSTEM RELOAD DICTIONARY "seqrdb_excluded_variant_dict"'), + migrations.RunSQL('SYSTEM RELOAD DICTIONARY "seqrdb_individual_metadata_dict"'), + ] diff --git a/clickhouse_search/models/postgres_dicts.py b/clickhouse_search/models/postgres_dicts.py index defeeb231d..6887b2b8d3 100644 --- a/clickhouse_search/models/postgres_dicts.py +++ b/clickhouse_search/models/postgres_dicts.py @@ -37,3 +37,53 @@ class Meta: layout = 'HASHED()' postgres_db = 'reference_data' postgres_query = 'SELECT gene_id, id FROM reference_data_geneinfo' + + +class IndividualMetadataDict(Dictionary): + family_guid = models.StringField(primary_key=True) + sampleId = models.StringField() + restrict_sharing = models.BoolField() + features = models.StringField() + omim_id = models.UInt32Field() + mondo_id = models.StringField() + is_solved = models.BoolField() + vlm_contact_email = models.StringField() + + class Meta: + db_table = 'seqrdb_individual_metadata_dict' + engine = models.MergeTree(primary_key=('family_guid', 'sampleId')) + layout = 'COMPLEX_KEY_HASHED()' + postgres_query = "select f.guid as family_guid, i.individual_id as sampleId, p.restrict_sharing as restrict_sharing, i.features as features, f.post_discovery_omim_numbers[1] as omim_id, f.post_discovery_mondo_id as mondo_id, f.analysis_status in (''S'', ''S_kgfp'', ''S_kgdp'', ''S_ng'', ''ES'') as is_solved, p.vlm_contact_email as vlm_contact_email FROM seqr_individual i INNER JOIN seqr_family f ON i.family_id = f.id INNER JOIN seqr_project p ON f.project_id = p.id" + + +class VariantFamiliesDict(Dictionary): + key = models.UInt32Field(primary_key=True) + dataset_type = models.StringField() + family_guids = models.ArrayField(models.StringField()) + + class Meta: + abstract = True + + @classmethod + def dict_get_expression(cls, *expressions, dataset_type=None, **kwargs): + return super().dict_get_expression( + *expressions, key_template=f"(%(expressions)s, '{dataset_type}')", field_names=['family_guids'], **kwargs, + ) + + +class DiscoveryVariantDict(VariantFamiliesDict): + + class Meta: + db_table = 'seqrdb_discovery_variant_dict' + engine = models.MergeTree(primary_key=('key', 'dataset_type')) + layout = 'COMPLEX_KEY_HASHED()' + postgres_query = "SELECT sv.key as key, sv.dataset_type as dataset_type, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.category = ''CMG Discovery Tags'') GROUP BY sv.key, sv.dataset_type" + + +class ExcludedVariantDict(VariantFamiliesDict): + + class Meta: + db_table = 'seqrdb_excluded_variant_dict' + engine = models.MergeTree(primary_key=('key', 'dataset_type')) + layout = 'COMPLEX_KEY_HASHED()' + postgres_query = "SELECT sv.key as key, sv.dataset_type as dataset_type, array_agg(distinct f.guid) as family_guids FROM seqr_savedvariant sv INNER JOIN seqr_family f ON sv.family_id = f.id WHERE sv.id IN (SELECT savedvariant_id FROM seqr_varianttag_saved_variants vts LEFT JOIN seqr_varianttag vt ON vts.varianttag_id = vt.id LEFT JOIN seqr_varianttagtype vtt ON vt.variant_tag_type_id = vtt.id WHERE vtt.name = ''Excluded'') GROUP BY sv.key, sv.dataset_type" diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 2c45bec171..3ddac5a2a9 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -1,4 +1,4 @@ -from clickhouse_backend.models import ArrayField, StringField +from clickhouse_backend.models import ArrayField, BoolField, StringField, UInt32Field from collections import defaultdict from django.contrib.postgres.aggregates import ArrayAgg from django.core.exceptions import ObjectDoesNotExist @@ -14,6 +14,7 @@ ArrayMap, Modulo from clickhouse_search.managers import InvalidDatasetTypeException, InvalidSearchException from clickhouse_search.models.gt_stats_models import PROJECT_GT_STATS_VIEW_CLASS_MAP +from clickhouse_search.models.postgres_dicts import SexDict, AffectedDict, IndividualMetadataDict, ExcludedVariantDict from clickhouse_search.models.reference_data_models import BaseClinvar, BaseHgmd from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, EntriesSnvIndel, \ ENTRY_CLASS_MAP, VARIANTS_CLASS_MAP, VARIANT_DETAILS_CLASS_MAP @@ -323,7 +324,7 @@ def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_d exclude_key_pairs=exclude_key_pairs.get(f'{Dataset.DATASET_TYPE_VARIANT_CALLS},{sv_dataset_type}'), ) dataset_results = _evaluate_results(result_q, is_comp_het=True) - if not sv_sample_data['samples']: + if not (sv_sample_data['samples'] and type_snv_indel_sample_data['samples']): _add_individual_guids(dataset_results) results += dataset_results searched_dataset_types.add(sv_dataset_type) @@ -469,7 +470,7 @@ def _set_individual_guids(result, sample_map): for genotype in genotypes: individual_guid = sample_map[(family_guid, genotype['sampleId'])] individual_genotypes[individual_guid].append({**genotype, 'individualGuid': individual_guid}) - result['genotypes'] = {k: v[0] if len(v) == 1 else v for k, v in individual_genotypes.items()} + result['genotypes'] = {k: v[0] if len(v) == 1 else sorted(v, key=lambda g: g.get('sampleType')) for k, v in individual_genotypes.items()} def get_sorted_search_results(results, sort, families): @@ -482,20 +483,12 @@ def get_sorted_search_results(results, sort, families): def format_clickhouse_export_results(results): - formatted_results = [variant for result in results for variant in (result if isinstance(result, list) else [result])] - if not formatted_results: - return [] - - genome_version = formatted_results[0]['genomeVersion'] - keys_with_no_details = {result['key'] for result in formatted_results if not 'transcripts' in result} - detail_qs = get_variant_details_queryset(genome_version, Dataset.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details) - details_by_key = { - detail['key']: detail for detail in detail_qs.values( - 'key', 'rsid', mainTranscript=F('transcripts__0'), variantId=F('variant_id'), - **detail_qs.split_variant_id_annotations(), - ) - } + details_by_key = _get_details_by_key(results, lambda detail_qs: detail_qs.values( + 'key', 'rsid', mainTranscript=F('transcripts__0'), variantId=F('variant_id'), + **detail_qs.split_variant_id_annotations(), + )) + formatted_results = [variant for result in results for variant in (result if isinstance(result, list) else [result])] gene_ids = set() for result in formatted_results: if 'transcripts' in result: @@ -513,19 +506,24 @@ def format_clickhouse_export_results(results): return formatted_results -def format_clickhouse_results(results): +def _get_details_by_key(results, format_details): if not results: - return [] + return {} genome_version = (results[0] if isinstance(results[0], list) else results)[0]['genomeVersion'] keys_with_no_details = { - variant['key'] for result in results for variant in (result if isinstance(result, list) else [result]) if not 'transcripts' in variant + variant['key'] for result in results for variant in (result if isinstance(result, list) else [result]) if + not 'transcripts' in variant } - details_by_key = { + return { detail['key']: detail for detail in - get_variant_details_queryset(genome_version, Dataset.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details).result_values() + format_details(get_variant_details_queryset(genome_version, Dataset.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details)) } + +def format_clickhouse_results(results): + details_by_key = _get_details_by_key(results, lambda detail_qs: detail_qs.result_values()) + formatted_results = [] for variant in results: if isinstance(variant, list): @@ -699,7 +697,7 @@ def _add_missing_multi_type_samples(individuals, data): def _no_affected_male_families(sample_data, user): sample_type_families = { - sample_type: families - set(sample_data['affected_male_family_guids']) + sample_type: set(families) - set(sample_data['affected_male_family_guids']) for sample_type, families in sample_data['sample_type_families'].items() } num_families = len(set().union(*sample_type_families.values())) @@ -715,7 +713,7 @@ def _affected_male_families(sample_data, affected_male_family_guids): if len(affected_male_family_guids) == sample_data['num_families']: return sample_data sample_type_families = { - sample_type: families.intersection(affected_male_family_guids) + sample_type: set(families).intersection(affected_male_family_guids) for sample_type, families in sample_data['sample_type_families'].items() } return { @@ -831,7 +829,7 @@ def _sv_size(x): 'alphamissense': [ lambda x: -max(t.get('alphamissensePathogenicity') or MIN_SORT_RANK for t in x[TRANSCRIPT_CONSEQUENCES_FIELD]) if x.get(TRANSCRIPT_CONSEQUENCES_FIELD) else MIN_SORT_RANK, ] + _subfield_sort(SELECTED_TRANSCRIPT_FIELD, 'alphamissensePathogenicity', reverse=True, default=MIN_SORT_RANK), - 'callset_af': _subfield_sort('populations', ('seqr', 'sv_callset'), 'ac'), + 'seqr_ac': _subfield_sort('populations', ('seqr', 'sv_callset'), 'ac'), 'family_guid': [lambda x: sorted(x.get('familyGuids', ['z']))[0]], 'gnomad': _subfield_sort('populations', ('gnomad_genomes', 'gnomad_mito', 'gnomad_svs'), 'af'), 'gnomad_exomes': _subfield_sort('populations', 'gnomad_exomes', 'af'), @@ -877,16 +875,28 @@ def _get_sort_key(sort, gene_metadata): return lambda x: tuple(expr(x[0] if isinstance(x, list) else x) for expr in [*sort_expressions, lambda x: x[XPOS_SORT_KEY]]) -def _clickhouse_variant_lookup(entries, genome_version, data_type, affected_only=False, hom_only=False): +def _clickhouse_variants_lookup(entries, genome_version, data_type, format_results, affected_only=False, hom_only=False): variants_cls = VARIANTS_CLASS_MAP[genome_version][data_type] entries = _filter_lookup_entries(entries, affected_only, hom_only) - entries = entries.result_values() + entries = entries.result_values(additional_expressions=_lookup_genotype_expressions()) results = variants_cls.objects.subquery_join(entries) + + return format_results(results).result_values(additional_values={ + 'excludedTagFamilies': ExcludedVariantDict.dict_get_expression('key', dataset_type=data_type), + }) + +def _add_results_override_annotations(results): if hasattr(results, 'add_genotype_override_annotations'): results = results.add_genotype_override_annotations(results) + return results + +def _clickhouse_variant_lookup(entries, genome_version, data_type, **kwargs): + results = _clickhouse_variants_lookup( + entries, genome_version, data_type, format_results=_add_results_override_annotations, **kwargs, + ) - variant = results.result_values().first() + variant = results.first() if variant: variant = format_clickhouse_results([variant])[0] return variant @@ -898,6 +908,25 @@ def _filter_lookup_entries(entries, affected_only, hom_only): entries = entries.filter(calls__array_exists={'gt': (2,)}) return entries +def _lookup_genotype_expressions(): + affected_expr = AffectedDict.dict_get_sql(key='(family_guid, x.sampleId)', fields=['affected'], default='U') + sex_expr = SexDict.dict_get_sql(key='(family_guid, x.sampleId)', fields=['sex'], default='U') + metadata_expr = IndividualMetadataDict.dict_get_sql( + key='(family_guid, x.sampleId)', fields=['restrict_sharing', 'is_solved', 'omim_id', 'mondo_id', 'features', 'vlm_contact_email'], + ) + return { + f'tupleConcat(({affected_expr}, {sex_expr}), {metadata_expr})': ('metadata', NamedTupleField([ + ('affected', StringField()), + ('sex', StringField()), + ('restrict_sharing', BoolField()), + ('isSolved', BoolField()), + ('omim_id', UInt32Field()), + ('mondo_id', StringField()), + ('features', StringField()), + ('vlmContactEmail', StringField()), + ])), + } + def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, affected_only, hom_only): data_type = entry_qs = None for dataset_type, entry_cls in sorted(ENTRY_CLASS_MAP[genome_version].items()): @@ -947,17 +976,19 @@ def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, aff (dt, cls) for dt, cls in ENTRY_CLASS_MAP[genome_version].items() if dt != data_type and dt.startswith(Dataset.DATASET_TYPE_SV_CALLS) ) - other_variants_cls = VARIANTS_CLASS_MAP[genome_version][other_sample_type] padding = int((variant['end'] - variant['pos']) * 0.2) + def format_results(results): + return results.search( + padded_interval_end=(variant['end'], padding), **results.get_parsed_annotations_filters( + annotations={'structural': [variant['svType'], f"gCNV_{variant['svType']}"]}, + ), + ) + entries = other_entry_class.objects.search_padded_interval(variant['chrom'], variant['pos'], padding) - entries = _filter_lookup_entries(entries, affected_only, hom_only) - results = other_variants_cls.objects.subquery_join(entries).search( - padded_interval_end=(variant['end'], padding), **other_variants_cls.objects.get_parsed_annotations_filters( - annotations={'structural': [variant['svType'], f"gCNV_{variant['svType']}"]}, - ), - ) - variants += list(results.result_values()) + variants += list(_clickhouse_variants_lookup( + entries, genome_version, other_sample_type, format_results, affected_only=affected_only, hom_only=hom_only, + )) return variants @@ -969,11 +1000,15 @@ def _add_liftover_genotypes(variant, data_type, affected_only, hom_only): lifted_id = f"{variant['liftedOverChrom']}-{variant['liftedOverPos']}-{variant['ref']}-{variant['alt']}" lifted_entries = lifted_entry_cls.objects.filter_locus(raw_variant_items=lifted_id) lifted_entries = _filter_lookup_entries(lifted_entries, affected_only, hom_only) - gt_field, gt_expr = lifted_entry_cls.objects.genotype_expression() - lifted_entry_data = lifted_entries.values('key').annotate(**{gt_field: GroupArrayArray(gt_expr)}) + gt_field, gt_expr = lifted_entry_cls.objects.genotype_expression(additional_expressions=_lookup_genotype_expressions()) + lifted_entry_data = lifted_entries.values('key').annotate( + excludedTagFamilies=ExcludedVariantDict.dict_get_expression('key', dataset_type=data_type), + **{gt_field: GroupArrayArray(gt_expr)}, + ) if lifted_entry_data: variant['familyGenotypes'].update(lifted_entry_data[0]['familyGenotypes']) variant['liftedFamilyGuids'] = sorted(lifted_entry_data[0]['familyGenotypes'].keys()) + variant['excludedTagFamilies'] += lifted_entry_data[0]['excludedTagFamilies'] def get_clickhouse_genotypes(project_guid, family_guids, genome_version, dataset_type, keys, additional_fields=None): @@ -1020,14 +1055,6 @@ def get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids): return lookup -def _main_transcript(selected_transcript_id, sorted_transcripts): - if not sorted_transcripts: - return {} - if selected_transcript_id: - return next((t for t in sorted_transcripts if t['transcriptId'] == selected_transcript_id), {}) - return sorted_transcripts[0] - - def delete_clickhouse_project(project, dataset_type, sample_type=None): if dataset_type == Dataset.DATASET_TYPE_SV_CALLS and sample_type == Dataset.SAMPLE_TYPE_WES: dataset_type = 'GCNV' @@ -1061,22 +1088,16 @@ def delete_clickhouse_project(project, dataset_type, sample_type=None): GENOME_VERSION_GRCh38: ('hg19', 'hg38'), GENOME_VERSION_GRCh37: ('hg38', 'hg19'), } -def _get_liftover(genome_version): + +def _run_liftover(genome_version, chrom, pos): if not LIFTOVERS[genome_version]: try: LIFTOVERS[genome_version] = LiftOver(*PYLIFTOVER_BUILD_LOOKUP[genome_version]) except Exception as e: logger.error('ERROR: Unable to set up liftover. {}'.format(e), user=None) - return LIFTOVERS[genome_version] - + return None -def _run_liftover(genome_version, chrom, pos): - liftover = _get_liftover(genome_version) - if not liftover: - return None - lifted_coord = liftover.convert_coordinate( + lifted_coord = LIFTOVERS[genome_version].convert_coordinate( 'chr{}'.format(chrom.lstrip('chr')), int(pos) ) - if lifted_coord and lifted_coord[0]: - return (lifted_coord[0][0].lstrip('chr'), lifted_coord[0][1]) - return None + return (lifted_coord[0][0].lstrip('chr'), lifted_coord[0][1]) if lifted_coord and lifted_coord[0] else None diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index b43764cd73..fc60017d04 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1205,6 +1205,55 @@ 'CAID': None, } +DISCOVERY_VARIANT = { + 'key': 100, + 'variantId': '1-248367227-TC-T', + 'chrom': '1', + 'pos': 248367227, + 'ref': 'TC', + 'alt': 'T', + 'genomeVersion': '38', + 'liftedOverGenomeVersion': '37', + 'liftedOverChrom': None, + 'liftedOverPos': None, + 'xpos': 1248367227, + 'rsid': None, + 'familyGuids': ['F000002_2'], + 'genotypes': { + 'I000004_hg00731': { + 'sampleId': 'HG00731', 'sampleType': 'WGS', 'individualGuid': 'I000004_hg00731', + 'familyGuid': 'F000002_2', + 'numAlt': 0, 'dp': 16, 'gq': 48, 'ab': 1.0, 'filters': [], + }, + 'I000006_hg00733': { + 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000006_hg00733', + 'familyGuid': 'F000002_2', + 'numAlt': 1, 'dp': 49, 'gq': 99, 'ab': 0.65306, 'filters': [], + }, + }, + 'clinvar': None, + 'hgmd': None, + 'screenRegionType': None, + 'populations': { + 'seqr': {'ac': 5, 'hom': 1, 'ac_wes': 0, 'ac_wgs': 5, 'hom_wes': 0, 'hom_wgs': 1}, + 'seqr_affected': {'ac': 4, 'hom': 1}, + 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, + 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, + 'gnomad_genomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, + }, + 'predictions': { + 'cadd': None, 'eigen': None, 'fathmm': None, 'gnomad_noncoding': None, 'mpc': None, 'mut_pred': None, + 'primate_ai': None, 'splice_ai': None, 'splice_ai_consequence': None, 'vest': None, 'mut_taster': None, + 'polyphen': None, 'revel': None, 'sift': None, 'absplice': None, 'pext': None, 'promoter_ai': None, + }, + 'transcripts': {}, + 'sortedMotifFeatureConsequences': None, + 'sortedRegulatoryFeatureConsequences': None, + 'mainTranscriptId': None, + 'selectedMainTranscriptId': None, + 'CAID': None, +} + CACHED_CONSEQUENCES_BY_KEY = {1: [], 2: [{ 'alphamissensePathogenicity': 0.99779, 'canonical': 1, @@ -1356,16 +1405,17 @@ 'geneId': 'ENSG00000268903', 'isManeSelect': False, }], +100: [], } def format_cached_variant(variant): if variant['key'] not in CACHED_CONSEQUENCES_BY_KEY: - return variant + return {**variant, 'discoveryFamilies': []} return { **{k: v for k, v in variant.items() if k not in [ 'mainTranscriptId', 'selectedMainTranscriptId', 'transcripts', 'CAID', 'chrom', 'pos', 'ref', 'alt', 'liftedOverChrom', 'liftedOverPos', 'rsid', 'variantId', 'sortedMotifFeatureConsequences', - 'sortedRegulatoryFeatureConsequences', + 'sortedRegulatoryFeatureConsequences', 'noAccessDiscoveryFamilies', 'discoveryTags', ]}, 'sortedTranscriptConsequences': CACHED_CONSEQUENCES_BY_KEY[variant['key']], } diff --git a/deploy/LOCAL_DEVELOPMENT_INSTALL.md b/deploy/LOCAL_DEVELOPMENT_INSTALL.md index e793532d27..132ff8d986 100644 --- a/deploy/LOCAL_DEVELOPMENT_INSTALL.md +++ b/deploy/LOCAL_DEVELOPMENT_INSTALL.md @@ -18,7 +18,10 @@ After installation, run `psql -l` and if there is no user named `postgres`, run - [clickhouse](https://clickhouse.com/docs/install) - [redis](https://redis.io/topics/quickstart) -- [node/npm <14](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm). Note: more recent versions of `node` may not function are not officially supported. +- [node/npm <14](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) + - Note: more recent versions of `node` may not function are not officially supported. +For certain npm installations on mac, you may run into issues running this version of npm in your terminal - +see [this post](https://stackoverflow.com/a/67254340) for a workaround. Additionally, you will need to install all the infrastructure components required for the [helm deployment](https://github.com/broadinstitute/seqr-helm?tab=readme-ov-file#instructions-for-initial-deployment) diff --git a/matchmaker/matchmaker_utils.py b/matchmaker/matchmaker_utils.py index 4ed0cce103..5ce439083a 100644 --- a/matchmaker/matchmaker_utils.py +++ b/matchmaker/matchmaker_utils.py @@ -126,9 +126,7 @@ def _parse_mme_gene_variants(result, gene_symbols_to_ids): def get_gene_ids_for_feature(gene_feature, gene_symbols_to_ids): - gene_id = gene_feature.get('gene', {}).get('id') - if not gene_id: - return [] + gene_id = gene_feature['gene']['id'] if not gene_id.startswith('ENSG'): gene_ids = gene_symbols_to_ids.get(gene_feature['gene']['id'], []) else: @@ -185,7 +183,7 @@ def _submission_genes_to_external_genomic_features(submission): def _submission_gene_to_external_genomic_features(submission_gene, individual): variant = submission_gene.saved_variant chrom, pos = get_chrom_pos(variant.xpos) - genome_version = variant.saved_variant_json.get('genomeVersion', individual.family.project.genome_version) + genome_version = individual.family.project.genome_version feature = { 'gene': {'id': submission_gene.gene_id}, @@ -204,8 +202,7 @@ def _submission_gene_to_external_genomic_features(submission_gene, individual): _, end = get_chrom_pos(variant.xpos_end) feature['variant']['end'] = end - genotypes = variant.genotypes or variant.saved_variant_json.get('genotypes', {}) - genotype = genotypes.get(individual.guid) + genotype = variant.genotypes.get(individual.guid) if genotype and genotype.get('numAlt', -1) > 0: feature['zygosity'] = genotype['numAlt'] diff --git a/matchmaker/views/matchmaker_api.py b/matchmaker/views/matchmaker_api.py index c261a5ee7f..be0ea94e25 100644 --- a/matchmaker/views/matchmaker_api.py +++ b/matchmaker/views/matchmaker_api.py @@ -51,7 +51,7 @@ def get_individual_mme_matches(request, submission_guid): variants = get_json_for_saved_variants( SavedVariant.objects.filter(guid__in=variant_guids), additional_values={ - 'genomeVersion': Coalesce('saved_variant_json__genomeVersion', Value(project.genome_version), output_field=CharField()), + 'genomeVersion': Value(project.genome_version), 'selectedMainTranscript': F('main_transcript'), 'xposEnd': F('xpos_end'), }, diff --git a/panelapp/panelapp_utils.py b/panelapp/panelapp_utils.py index d4b8112e53..7394e28d05 100644 --- a/panelapp/panelapp_utils.py +++ b/panelapp/panelapp_utils.py @@ -14,32 +14,17 @@ class TooManyRequestsError(Exception): pass -def _extract_ensembl_id_from_json(raw_gene_json): - ensembl_genes_json = raw_gene_json.get('gene_data', {}).get('ensembl_genes') - if ensembl_genes_json and isinstance(ensembl_genes_json, dict): - return ensembl_genes_json \ - .get('GRch38', {}) \ - .get('90', {}) \ - .get('ensembl_id') - else: - return None - - def get_valid_panel_genes(panel_app_id, panel, panels_api_url, genes_by_panel_id, gene_ids_to_gene): if len(genes_by_panel_id[panel_app_id]) != panel['stats']['number_of_genes']: panel_genes_url = f'{panels_api_url}/{panel_app_id}/genes' _get_all_genes(panel_app_id, panel_genes_url, genes_by_panel_id) - all_genes_for_panel = genes_by_panel_id[panel_app_id] - if not all_genes_for_panel: - return {} - panel_genes_by_id = { - _extract_ensembl_id_from_json(gene): gene for gene in all_genes_for_panel - if _extract_ensembl_id_from_json(gene) + gene['gene_data']['ensembl_genes'].get('GRch38', {}).get('90', {}).get('ensembl_id'): gene + for gene in genes_by_panel_id[panel_app_id] if isinstance(gene.get('gene_data', {}).get('ensembl_genes'), dict) } valid_panel_genes = { - gene_id: panel_gene for gene_id, panel_gene in panel_genes_by_id.items() if gene_id in gene_ids_to_gene + gene_id: panel_gene for gene_id, panel_gene in panel_genes_by_id.items() if gene_id and gene_id in gene_ids_to_gene } if len(panel_genes_by_id) > len(valid_panel_genes): invalid_items = sorted(set(panel_genes_by_id.keys()) - set(valid_panel_genes.keys())) diff --git a/reference_data/models.py b/reference_data/models.py index 8aad05578c..e64a578852 100644 --- a/reference_data/models.py +++ b/reference_data/models.py @@ -209,22 +209,20 @@ def get_record_models(cls, records, **kwargs): model.category_id = cls._get_category_id(parent_id_map, model.hpo_id) return models - @staticmethod - def _get_category_id(parent_id_map, hpo_id): - if hpo_id == 'HP:0000001': + @classmethod + def _get_category_id(cls, parent_id_map, hpo_id): + if hpo_id == 'HP:0000001' or hpo_id is None: return None if hpo_id not in parent_id_map: - return None + raise ValueError('Strange id: %s' % hpo_id) - while hpo_id and parent_id_map.get(hpo_id) != 'HP:0000118': - if hpo_id not in parent_id_map: - raise ValueError('Strange id: %s' % hpo_id) - hpo_id = parent_id_map[hpo_id] - if hpo_id == 'HP:0000001': - return None + parent_hpo_id = parent_id_map[hpo_id] + if parent_hpo_id == 'HP:0000118': + return hpo_id + + return cls._get_category_id(parent_id_map, parent_hpo_id) - return hpo_id class GeneInfo(LoadableModel): ALL_GENCODE_VERSIONS = ['39', '31', '29', '28', '27', '19'] @@ -336,6 +334,8 @@ def update_records(cls, gencode_release=CURRENT_VERSION, existing_gene_ids=None, class GeneMetadataModel(LoadableModel): + RECORD_FIELDS = {} + gene = models.ForeignKey(GeneInfo, on_delete=models.CASCADE) class Meta: @@ -358,9 +358,12 @@ def parse_record(cls, record, skipped_genes=None, **kwargs): record = None yield record - @staticmethod - def parse_gene_record(record): - return record + @classmethod + def parse_gene_record(cls, record): + return { + field: format(record[record_field]) if format else record[record_field] + for field, (record_field, format) in cls.RECORD_FIELDS.items() + } @classmethod def update_records(cls, **kwargs): @@ -436,6 +439,13 @@ class GeneConstraint(GeneMetadataModel): CURRENT_VERSION = 'gnomad.v2.1.1.lof_metrics.by_gene' URL = f'http://storage.googleapis.com/seqr-reference-data/gene_constraint/{CURRENT_VERSION}.txt' + RECORD_FIELDS = { + 'gene_id': ('gene_id', lambda gene_id: gene_id.split(".")[0]), + 'gene_symbol': ('gene', None), + 'mis_z': ('mis_z', lambda value: float(value) if value != 'NaN' else -100), + 'pLI': ('pLI', lambda value: float(value) if value != 'NA' else 0), + 'louef': ('oe_lof_upper', lambda value: float(value) if value != 'NA' else 100), + } mis_z = models.FloatField() mis_z_rank = models.IntegerField() @@ -447,16 +457,6 @@ class GeneConstraint(GeneMetadataModel): class Meta: json_fields = ['mis_z', 'mis_z_rank', 'pLI', 'pLI_rank', 'louef', 'louef_rank'] - @staticmethod - def parse_gene_record(record): - return { - 'gene_id': record['gene_id'].split(".")[0], - 'gene_symbol': record['gene'], - 'mis_z': float(record['mis_z']) if record['mis_z'] != 'NaN' else -100, - 'pLI': float(record['pLI']) if record['pLI'] != 'NA' else 0, - 'louef': float(record['oe_lof_upper']) if record['oe_lof'] != 'NA' else 100, - } - @classmethod def get_record_models(cls, records, **kwargs): # add _rank fields @@ -470,6 +470,11 @@ class GeneCopyNumberSensitivity(GeneMetadataModel): CURRENT_VERSION = 'Collins_rCNV_2022' URL = f'https://zenodo.org/record/6347673/files/{CURRENT_VERSION}.dosage_sensitivity_scores.tsv.gz' + RECORD_FIELDS = { + 'gene_symbol': ('#gene', None), + 'pHI': ('pHaplo', float), + 'pTS': ('pTriplo', float), + } pHI = models.FloatField() pTS = models.FloatField() @@ -477,32 +482,21 @@ class GeneCopyNumberSensitivity(GeneMetadataModel): class Meta: json_fields = ['pHI', 'pTS'] - @staticmethod - def parse_gene_record(record): - return { - 'gene_symbol': record['#gene'], - 'pHI': float(record['pHaplo']), - 'pTS': float(record['pTriplo']), - } - class GeneShet(GeneMetadataModel): CURRENT_VERSION = '7939768' URL = f'https://zenodo.org/record/{CURRENT_VERSION}/files/s_het_estimates.genebayes.tsv' + RECORD_FIELDS = { + 'gene_id': ('ensg', None), + 'post_mean': ('post_mean', float), + } post_mean = models.FloatField() class Meta: json_fields = ['post_mean'] - @staticmethod - def parse_gene_record(record): - return { - 'gene_id': record['ensg'], - 'post_mean': float(record['post_mean']), - } - class Omim(LoadableModel): @@ -682,8 +676,8 @@ class dbNSFPGene(GeneMetadataModel): class Meta: json_fields = ['function_desc', 'disease_desc', 'gene_names'] - @staticmethod - def parse_gene_record(record): + @classmethod + def parse_gene_record(cls, record): parsed_record = {DBNSFP_FIELD_MAP.get(k, k.split('(')[0].lower()): (v if v != '.' else '') for k, v in record.items() if not k.startswith(DBNSFP_EXCLUDE_FIELDS)} parsed_record["function_desc"] = parsed_record["function_desc"].replace("FUNCTION: ", "") @@ -703,6 +697,11 @@ class PrimateAI(GeneMetadataModel): CURRENT_VERSION = 'cleaned_v0.2' URL = f'http://storage.googleapis.com/seqr-reference-data/primate_ai/Gene_metrics_clinvar_pcnt.{CURRENT_VERSION}.txt' + RECORD_FIELDS = { + 'gene_symbol': ('genesymbol', None), + 'percentile_25': ('pcnt25', float), + 'percentile_75': ('pcnt75', float), + } percentile_25 = models.FloatField() percentile_75 = models.FloatField() @@ -710,19 +709,12 @@ class PrimateAI(GeneMetadataModel): class Meta: json_fields = ['percentile_25', 'percentile_75'] - @staticmethod - def parse_gene_record(record): - return { - 'gene_symbol': record['genesymbol'], - 'percentile_25': float(record['pcnt25']), - 'percentile_75': float(record['pcnt75']), - } - class MGI(GeneMetadataModel): CURRENT_VERSION = 'HMD_HumanPhenotype' URL = f'https://storage.googleapis.com/seqr-reference-data/mgi/{CURRENT_VERSION}.rpt.txt' + RECORD_FIELDS = {k: (k, lambda v: v.strip()) for k in ['gene_symbol', 'marker_id', 'entrez_gene_id']} marker_id = models.CharField(max_length=15) @@ -734,10 +726,6 @@ class Meta: def get_file_header(f): return ['gene_symbol', 'entrez_gene_id', 'mouse_gene_symbol', 'marker_id', 'phenotype_ids'] - @staticmethod - def parse_gene_record(record): - return {k: v.strip() for k, v in record.items() if k in ['gene_symbol', 'marker_id', 'entrez_gene_id']} - @classmethod def update_records(cls, **kwargs): entrez_id_to_gene = dict(dbNSFPGene.objects.values_list('entrez_gene_id', 'gene_id')) @@ -783,12 +771,12 @@ def get_file_header(f): def get_file_iterator(cls, f): return super().get_file_iterator(csv.reader(f)) - @staticmethod - def parse_gene_record(record): + @classmethod + def parse_gene_record(cls, record): return { 'gene_symbol': record['gene_symbol'], 'hgnc_id': record['gene_curie'], - 'classifications': [{title: record[field] for field, title in GenCC.CLASSIFICATION_FIELDS.items()}] + 'classifications': [{title: record[field] for field, title in cls.CLASSIFICATION_FIELDS.items()}] } @classmethod @@ -807,6 +795,12 @@ def get_record_models(cls, records, **kwargs): class ClinGen(GeneMetadataModel): URL = 'https://search.clinicalgenome.org/kb/gene-dosage/download' + RECORD_FIELDS = { + 'gene_symbol': ('gene_symbol', None), + 'haploinsufficiency': ('haploinsufficiency', lambda value: value.replace(' for Haploinsufficiency', '')), + 'triplosensitivity': ('triplosensitivity', lambda value: value.replace(' for Triplosensitivity', '')), + 'href': ('online_report', None), + } haploinsufficiency = models.TextField() triplosensitivity = models.TextField() @@ -836,14 +830,6 @@ def get_file_header(f): def get_file_iterator(cls, f): return super().get_file_iterator(csv.reader(f)) - @staticmethod - def parse_gene_record(record): - return { - 'gene_symbol': record['gene_symbol'], - 'haploinsufficiency': record['haploinsufficiency'].replace(' for Haploinsufficiency', ''), - 'triplosensitivity': record['triplosensitivity'].replace(' for Triplosensitivity', ''), - 'href': record['online_report'], - } class DataVersions(models.Model): data_model_name = models.CharField(max_length=30, primary_key=True) diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index 4761736bdf..144c9cf2d5 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -377,7 +377,7 @@ "last_modified_date": "2017-03-12T22:37:17.555Z", "project": 4, "family_id": "fam14", - "analysis_status": "Rncc", + "analysis_status": "S_ng", "success_story": "Differential treatement", "success_story_types": ["A", "D"] } @@ -1472,6 +1472,7 @@ "last_modified_date": "2017-03-13T09:07:49.744Z", "individual": 7, "sample_type": "alignment", + "index_file_path": "gs://readviz/NA20870.cram.crai", "file_path": "gs://readviz/NA20870.cram" } }, @@ -1616,31 +1617,6 @@ "gene_ids": ["ENSG00000135953"], "main_transcript": {"transcriptId": "ENST00000258436", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "MFSD9", "majorConsequence": "inframe_deletion", "canonical": "YES", "hgvsp": "ENSP00000258436.5:p.Leu126del", "lof": "", "lofFlags": "", "codons": "ctTCTc/ctc", "hgvsc": "ENST00000258436.5:c.375_377delTCT", "transcriptRank": 0, "geneId": "ENSG00000135953", "aminoAcids": "LL/L", "cdnaPosition": "419-421"}, "genotypes": {"I000003_na19679": {"sampleId": "NA19679", "ab": 0.0, "ad": "45,0", "gq": 99.0, "dp": "45", "pl": "0,135,1525", "cnvs": {"size": null, "snps": null, "cn": null, "LRR_sd": null, "array": null, "caller": null, "type": null, "freq": null, "LRR_median": null}, "numAlt": 0}, "I000002_na19678": {"sampleId": "NA19678", "ab": 0.0, "ad": "42,0", "gq": 99.0, "dp": "43", "pl": "0,126,1479", "cnvs": {"size": null, "snps": null, "cn": null, "LRR_sd": null, "array": null, "caller": null, "type": null, "freq": null, "LRR_median": null}, "numAlt": 0}, "I000001_na19675": {"sampleId": "NA19675_1", "ab": 0.7021276595744681, "ad": "14,33", "gq": 46.0, "dp": "50", "pl": "46,0,686", "cnvs": {"size": null, "snps": null, "cn": null, "LRR_sd": null, "array": null, "caller": null, "type": null, "freq": null, "LRR_median": null}, "numAlt": 1}}, - "saved_variant_json": { - "variantId": "21-3343353-GAGA-G", - "clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null}, - "liftedOverGenomeVersion": "", - "familyGuids": ["F000001_1", "F000002_2"], - "liftedOverPos": "", - "mainTranscriptId": "ENST00000258436", - "populations": {"callset": {"ac": null, "an": null, "af": null}, "g1k": {"ac": null, "an": null, "af": 0.0}, "gnomad_genomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.0}, "gnomad_exomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 2.4418633044922146e-05}, "exac": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.000242306760358614}, "topmed": {"ac": null, "an": null, "af": null}}, - "genomeVersion": "37", - "pos": 3343353, - "predictions": {"eigen": null, "revel": null, "sift": null, "cadd": "14.33", "metasvm": "", "mpc": null, "splice_ai": null, "phastcons_100_vert": null, "mut_taster": null, "fathmm": null, "polyphen": null, "dann": null, "primate_ai": null, "gerp_rs": null}, - "hgmd": {"accession": null, "class": null}, - "rsid": null, - "selectedMainTranscriptId": null, - "liftedOverChrom": "", - "transcripts": {"ENSG00000135953": [ - {"transcriptId": "ENST00000258436", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "MFSD9", "majorConsequence": "inframe_deletion", "canonical": "YES", "hgvsp": "ENSP00000258436.5:p.Leu126del", "lof": "", "lofFlags": "", "codons": "ctTCTc/ctc", "hgvsc": "ENST00000258436.5:c.375_377delTCT", "transcriptRank": 0, "geneId": "ENSG00000135953", "aminoAcids": "LL/L", "cdnaPosition": "419-421"}, - {"transcriptId": "ENST00000411991", "lofFilter": "", "biotype": "nonsense_mediated_decay", "geneSymbol": "MFSD9", "majorConsequence": "3_prime_UTR_variant", "canonical": "", "hgvsp": "", "lof": "", "lofFlags": "", "codons": "", "hgvsc": "ENST00000411991.1:c.*211_*213delTCT", "transcriptRank": 100, "geneId": "ENSG00000135953", "aminoAcids": "", "cdnaPosition": "558-560"}, - {"transcriptId": "ENST00000421966", "lofFilter": "", "biotype": "nonsense_mediated_decay", "geneSymbol": "MFSD9", "majorConsequence": "downstream_gene_variant", "canonical": "", "hgvsp": "", "lof": "", "lofFlags": "", "codons": "", "hgvsc": "", "transcriptRank": 100, "geneId": "ENSG00000135953", "aminoAcids": "", "cdnaPosition": null}, {"transcriptId": "ENST00000428085", "lofFilter": "", "biotype": "nonsense_mediated_decay", "geneSymbol": "MFSD9", "majorConsequence": "inframe_deletion", "canonical": "", "hgvsp": "ENSP00000413641.1:p.Leu48del", "lof": "", "lofFlags": "", "codons": "ctTCTc/ctc", "hgvsc": "ENST00000428085.1:c.141_143delTCT", "transcriptRank": 100, "geneId": "ENSG00000135953", "aminoAcids": "LL/L", "cdnaPosition": "141-143"}, - {"transcriptId": "ENST00000437075", "lofFilter": "", "biotype": "nonsense_mediated_decay", "geneSymbol": "MFSD9", "majorConsequence": "3_prime_UTR_variant", "canonical": "", "hgvsp": "", "lof": "", "lofFlags": "", "codons": "", "hgvsc": "ENST00000437075.2:c.*176_*178delTCT", "transcriptRank": 100, "geneId": "ENSG00000135953", "aminoAcids": "", "cdnaPosition": "541-543"}, - {"transcriptId": "ENST00000438943", "lofFilter": "", "biotype": "nonsense_mediated_decay", "geneSymbol": "MFSD9", "majorConsequence": "3_prime_UTR_variant", "canonical": "", "hgvsp": "", "lof": "", "lofFlags": "", "codons": "", "hgvsc": "ENST00000438943.1:c.*211_*213delTCT", "transcriptRank": 100, "geneId": "ENSG00000135953", "aminoAcids": "", "cdnaPosition": "558-560"}]}, - "chrom": "21", - "genotypes": {"I000003_na19679": {"sampleId": "NA19679", "ab": 0.0, "ad": "45,0", "gq": 99.0, "dp": "45", "pl": "0,135,1525", "cnvs": {"size": null, "snps": null, "cn": null, "LRR_sd": null, "array": null, "caller": null, "type": null, "freq": null, "LRR_median": null}, "numAlt": 0}, "I000002_na19678": {"sampleId": "NA19678", "ab": 0.0, "ad": "42,0", "gq": 99.0, "dp": "43", "pl": "0,126,1479", "cnvs": {"size": null, "snps": null, "cn": null, "LRR_sd": null, "array": null, "caller": null, "type": null, "freq": null, "LRR_median": null}, "numAlt": 0}, "I000001_na19675": {"sampleId": "NA19675_1", "ab": 0.7021276595744681, "ad": "14,33", "gq": 46.0, "dp": "50", "pl": "46,0,686", "cnvs": {"size": null, "snps": null, "cn": null, "LRR_sd": null, "array": null, "caller": null, "type": null, "freq": null, "LRR_median": null}, "numAlt": 1}}, - "CAID": null - }, "family": 1 } }, @@ -1668,7 +1644,6 @@ } }, "gene_ids": ["ENSG00000240361", "ENSG00000135953"], - "saved_variant_json": {}, "main_transcript": {"aminoAcids": null, "biotype": "protein_coding", "canonical": 1, "codons": null, "consequenceTerms": [], "geneId": "ENSG00000135953", "hgvsc": null, "hgvsp": null, "loftee": [null, []], "majorConsequence": null, "transcriptId": "ENST00000371839", "transcriptRank": 1}, "selected_main_transcript_id": "ENST00000371839", "family": 2 @@ -1695,30 +1670,6 @@ "I000002_na19678": {"sampleId": "NA19678", "ab": 0, "ad": null, "gq": 6, "dp": 2, "pl": null, "numAlt": 0}, "I000002_na19675": {"sampleId": "NA19675", "ab": 0.125, "ad": null, "gq": 7, "dp": 8, "pl": null, "numAlt": 1} }, - "saved_variant_json": { - "clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null}, - "liftedOverGenomeVersion": "38", - "familyGuids": ["F000001_1"], - "liftedOverPos": "", - "mainTranscriptId": "ENST00000505820", - "populations": {"callset": {"ac": null, "an": null, "af": null}, "g1k": {"ac": null, "an": null, "af": 0.0}, "gnomad_genomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.0}, "gnomad_exomes": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 8.142526788913136e-06}, "exac": {"hemi": null, "ac": null, "an": null, "hom": null, "af": 0.0}, "topmed": {"ac": null, "an": null, "af": null}}, - "pos": 1560662, - "predictions": {"eigen": null, "revel": null, "sift": "damaging", "cadd": "31", "metasvm": "D", "mpc": null, "splice_ai": null, "phastcons_100_vert": null, "mut_taster": "disease_causing", "fathmm": "tolerated", "polyphen": "probably_damaging", "dann": null, "primate_ai": null, "gerp_rs": null}, - "hgmd": {"accession": null, "class": null}, - "rsid": null, - "liftedOverChrom": "", - "transcripts": { - "ENSG00000197530": [ - {"transcriptId": "ENST00000505820", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "CELSR1", "majorConsequence": "splice_region_variant", "canonical": "YES", "hgvs": "ENST00000505820.2:c.1067-4G>A", "lof": "", "lofFlags": "", "codons": "Gtg/Atg", "hgvsc": "ENST00000505820.2:c.1067-4G>A", "transcriptRank": 0, "geneId": "ENSG00000197530", "aminoAcids": "V/M", "cdnaPosition": "3955"} - ] - }, - "chrom": "1", - "genotypes": { - "I000003_na19679": {"sampleId": "NA19679", "ab": 0, "ad": null, "gq": 21, "dp": 7, "pl": null, "numAlt": 0}, - "I000002_na19678": {"sampleId": "NA19678", "ab": 0, "ad": null, "gq": 6, "dp": 2, "pl": null, "numAlt": 0}, - "I000002_na19675": {"sampleId": "NA19675", "ab": 0.125, "ad": null, "gq": 7, "dp": 8, "pl": null, "numAlt": 1} - } - }, "family": 1 } }, @@ -1743,74 +1694,6 @@ "I000002_na19678": {"sampleId": "NA19678", "ab": 0, "ad": null, "gq": 30, "dp": 10, "pl": null, "numAlt": 0}, "I000002_na19675": {"sampleId": "NA19675", "ab": 0.5555556, "ad": null, "gq": 99, "dp": 9, "pl": null, "numAlt": 1} }, - "saved_variant_json": { - "clinvar": {"clinicalSignificance": "", "alleleId": null, "variationId": null, "goldStars": null}, - "liftedOverGenomeVersion": "38", - "familyGuids": ["F000001_1"], - "liftedOverPos": "", - "mainTranscriptId": "ENST00000505820", - "populations": { - "callset": { - "ac": 11, - "af": 0.344, - "an": 32, - "hemi": null, - "hom": null - }, - "exac": { - "ac": 28869, - "af": 0.6478622327790974, - "an": 80512, - "hemi": 0, - "hom": 5122 - }, - "g1k": { - "ac": 2206, - "af": 0.6838, - "an": 5008, - "hemi": null, - "hom": null - }, - "gnomad_exomes": { - "ac": 67263, - "af": 0.6339977851605758, - "an": 216284, - "hemi": 0, - "hom": 12201 - }, - "gnomad_genomes": { - "ac": 10595, - "af": 0.6165093253511398, - "an": 30852, - "hemi": 0, - "hom": 2311 - }, - "topmed": { - "ac": 45912, - "af": 0.365635, - "an": 125568, - "hemi": null, - "hom": 10332 - } - }, - "genomeVersion": "38", - "pos": 1562437, - "hgmd": {"accession": null, "class": null}, - "rsid": null, - "liftedOverChrom": "", - "transcripts": { - "ENSG00000197530": [ - {"transcriptId": "ENST00000505820", "lofFilter": "", "biotype": "protein_coding", "geneSymbol": "MIB2", "majorConsequence": "intron_variant", "canonical": 1, "hgvsp": "ENST00000505820.2:c.1586-17C>G", "lof": "", "lofFlags": "", "codons": "Gtg/Atg", "hgvsc": "ENST00000262738.3:c.3955G>A", "transcriptRank": 0, "geneId": "ENSG00000197530", "aminoAcids": "V/M", "cdnaPosition": "3955"} - ] - }, - "chrom": "1", - "genotypes": { - "I000003_na19679": {"sampleId": "NA19679", "ab": 0.71428573, "ad": null, "gq": 58, "dp": 7, "pl": null, "numAlt": 1}, - "I000002_na19678": {"sampleId": "NA19678", "ab": 0, "ad": null, "gq": 30, "dp": 10, "pl": null, "numAlt": 0}, - "I000002_na19675": {"sampleId": "NA19675", "ab": 0.5555556, "ad": null, "gq": 99, "dp": 9, "pl": null, "numAlt": 1} - }, - "CAID": null - }, "family": 1 } }, @@ -1835,7 +1718,6 @@ }, "gene_ids": ["ENSG00000240361", "ENSG00000135953"], "main_transcript": {"aminoAcids": null, "biotype": "protein_coding", "canonical": 1, "codons": "Gtg/Atg", "consequenceTerms": ["intron_variant"], "geneId": "ENSG00000240361", "hgvsc": "ENST00000262738.3:c.3955G>A", "hgvsp": "ENST00000505820.2:c.1586-17C>G", "loftee": [null, []], "majorConsequence": "intron_variant", "transcriptId": "ENST00000505820", "transcriptRank": 0}, - "saved_variant_json": {}, "family": 12 } }, @@ -1858,28 +1740,6 @@ "genotypes": { "I000017_na20889": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} }, - "saved_variant_json": { - "liftedOverGenomeVersion": null, - "pos": 249045487, - "end": 249045898, - "xpos": 1249045487, - "predictions": {"strvctvre": 0.374}, - "alt": null, - "numExon": 2, - "genotypeFilters": [], - "ref": null, - "genotypes": { - "I000017_na20889": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} - }, - "liftedOverPos": null, - "liftedOverChrom": null, - "svType": "DEL", - "variantId": "prefix_19107_DEL", - "chrom": "1", - "genomeVersion": "37", - "populations": {"sv_callset": {}}, - "transcripts": {"ENSG00000240361": [], "ENSG00000135953": [], "ENSG00000223972": []} - }, "family": 12 } }, @@ -1901,7 +1761,6 @@ "genotypes": { "I000018_na21234": {"sampleId": "NA20885", "ab": 0.0, "gq": 99.0, "numAlt": 1} }, - "saved_variant_json": {}, "family": 14 } }, diff --git a/seqr/fixtures/report_variants.json b/seqr/fixtures/report_variants.json index 3571f6b9c2..b35a39fce8 100644 --- a/seqr/fixtures/report_variants.json +++ b/seqr/fixtures/report_variants.json @@ -17,35 +17,6 @@ "genotypes": { "I000004_hg00731": {"sampleId": "HG00731", "ab": 0, "ad": null, "gq": 21, "dp": 7, "pl": null, "numAlt": 1} }, - "saved_variant_json": { - "liftedOverGenomeVersion": "37", - "pos": 1912634, - "end": 1912634, - "xpos": 19001912634, - "ref": "C", - "alt": "T", - "genomeVersion": "38", "genotypeFilters": [], - "genotypes": { - "I000004_hg00731": {"sampleId": "HG00731", "ab": 0, "ad": null, "gq": 21, "dp": 7, "pl": null, "numAlt": 1} - }, - "populations": { - "g1k": { - "ac": 2206, - "af": 0.6838, - "an": 5008, - "hemi": null, - "hom": null - } - }, - "liftedOverPos": 1912633, - "liftedOverChrom": "19", - "variantId": "19-1912634-C-T", - "chrom": "19", - "transcripts": { - "ENSG00000240361": [] - }, - "CAID": "CA403171634" - }, "family": 2 } }, @@ -68,38 +39,6 @@ "genotypes": { "I000004_hg00731": {"sampleId": "HG00731", "ab": 0, "ad": null, "gq": 21, "dp": 7, "pl": null, "numAlt": 1} }, - "saved_variant_json": { - "liftedOverGenomeVersion": "37", - "pos": 1912633, - "end": 1912633, - "xpos": 19001912633, - "ref": "G", - "alt": "T", - "genomeVersion": "38", "genotypeFilters": [], - "genotypes": { - "I000004_hg00731": {"sampleId": "HG00731", "ab": 0, "ad": null, "gq": 21, "dp": 7, "pl": null, "numAlt": 1} - }, - "populations": { - "g1k": { - "ac": 2206, - "af": 0.6838, - "an": 5008, - "hemi": null, - "hom": null - } - }, - "liftedOverPos": 1912632, - "liftedOverChrom": "19", - "variantId": "19-1912633-G-T", - "chrom": "19", - "mainTranscriptId": "ENST00000371839", - "transcripts": { - "ENSG00000135953": [ - {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000240361"} - ] - }, - "CAID": "CA403171631" - }, "family": 2 } }, @@ -122,25 +61,6 @@ "genotypes": { "I000004_hg00731": {"numAlt": 1} }, - "saved_variant_json": { - "pos": 1912632, - "end": 1912632, - "xpos": 19001912632, - "genomeVersion": "38", "genotypeFilters": [], - "genotypes": { - "I000004_hg00731": {"numAlt": 1} - }, - "variantId": "19-1912632-G-C", - "chrom": "19", - "mainTranscriptId": "ENST00000371839", - "transcripts": { - "ENSG00000135953": [ - {"transcriptId": "ENST00000371839", "biotype": "protein_coding", "geneId": "ENSG00000240361", - "hgvsc": "c.586_587delinsTT", "hgvsp": "p.Ala196Leu"} - ] - }, - "CAID": null - }, "family": 2 } }, @@ -164,7 +84,6 @@ }, "gene_ids": ["ENSG00000198727"], "main_transcript": {"aminoAcids": "L", "biotype": "protein_coding", "canonical": 1, "codons": "Tta/Cta", "consequenceTerms": ["synonymous_variant"], "geneId": "ENSG00000198727", "hgvsc": "ENST00000361789.2:c.37T>C", "hgvsp": "ENSP00000354554.2:p.Leu13=", "loftee": [null, []], "majorConsequence": "synonymous_variant", "transcriptId": "ENST00000361789", "transcriptRank": 0}, - "saved_variant_json": {}, "family": 14 } }, { @@ -187,7 +106,6 @@ "I000018_na21234": { "cn": 1, "sampleId": "NA20885", "numAlt": -1, "defragged": false, "qs": 33, "numExon": 2} }, "gene_ids": ["ENSG00000240361", "ENSG00000135953", "ENSG00000223972"], - "saved_variant_json": {}, "family": 14 } }, diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 7ae407db96..57c66a8138 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -209,6 +209,7 @@ }, 'sex_check': { 'F000001_1': {'reasons': ['Sample NA19679 has pedigree sex F but imputed sex M']}, + 'F000011_11': {'reasons': ['Sample NA20870 has pedigree sex M but imputed sex F']}, 'F000014_14': {'reasons': ['Sample NA21987 has pedigree sex M but imputed sex F']}, }, 'missing_samples': { @@ -357,8 +358,9 @@ class CheckNewSamplesTest(object): 'updateType': 'bulk_update'}} ), ('Reloading saved variants in 2 projects', None), - ('Reloading genotypes for 0 SNV_INDEL variants in family F000012_12', None), - ('Updated 0 variants in 2 families for project Test Reprocessed Project', None), + ('Reloading genotypes for 1 SNV_INDEL variants in family F000012_12', None), + ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), + ('Updated 1 variants in 2 families for project Test Reprocessed Project', None), ('Reloading genotypes for 1 SNV_INDEL variants in family F000014_14', None), ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), ('Updated 1 variants in 1 families for project Non-Analyst Project', None), @@ -456,17 +458,20 @@ def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=6): num_calls = self._assert_expected_airtable_calls(bool(run_loading_logs), single_call) self.assertEqual(len(responses.calls), num_calls) - def _test_success_call(self, anvil_email_calls, next_dataset_id=155): + def _test_success_call(self, anvil_email_calls, next_dataset_id=155, fetched_tracking=True): Project.objects.filter(id__in=[1, 3]).update(genome_version=38) self.maxDiff = None + tracking_logs = [ + ('Fetched 2 AnVIL Seqr Loading Requests Tracking records from airtable', None), + ] if fetched_tracking and self.AIRTABLE_LOGS else [] self._test_call(run_loading_logs={ 'GRCh38/SNV_INDEL': [ ('Loading 4 WES SNV_INDEL samples in 2 projects', None), ('update 2 Familys', {'dbUpdate': mock.ANY}), (f'create Dataset D0000{next_dataset_id}_snv_indel_wes_2025_09', {'dbUpdate': mock.ANY}), (f'create Dataset D0000{next_dataset_id+1}_snv_indel_wes_2025_09', {'dbUpdate': mock.ANY}), - ] + self.AIRTABLE_LOGS + [ + ] + self.AIRTABLE_LOGS + tracking_logs + [ ('update 3 Familys', {'dbUpdate': mock.ANY}), ] + self.UPDATE_SAMPLE_LOGS, 'GRCh38/MITO': [ @@ -488,7 +493,7 @@ def _test_success_call(self, anvil_email_calls, next_dataset_id=155): }) # Test notifications - self.assertEqual(self.mock_send_slack.call_count, 7 + len(self.ADDITIONAL_SLACK_CALLS)) + self.assertEqual(self.mock_send_slack.call_count, 8 + len(self.ADDITIONAL_SLACK_CALLS)) self.mock_send_slack.assert_has_calls([ mock.call( 'seqr-data-loading', @@ -508,6 +513,13 @@ def _test_success_call(self, anvil_email_calls, next_dataset_id=155): The following 2 families failed missing samples: - 2: Missing samples: {'HG00732', 'HG00733'} - 3: Missing samples: {'NA20870'}""", + ), + mock.call( + 'seqr_loading_notifications', + f"""Encountered the following errors loading Test Reprocessed Project: + +The following 1 families failed sex check: +- 11: Sample NA20870 has pedigree sex M but imputed sex F{self.SKIPPED_PDO_MESSAGE}""", ), mock.call( 'seqr_loading_notifications', @@ -594,12 +606,6 @@ def test_command(self): self._test_call(error_logs=error_logs) self.assertEqual(Dataset.objects.filter(guid__in=NEW_DATASET_GUIDS + NEW_GCNV_DATASET_GUIDS).count(), 0) - # Update fixture data to allow testing edge cases - svs = SavedVariant.objects.filter(guid__in=['SV0000002_1248367227_r0390_100', 'SV0000006_1248367227_r0003_tes', 'SV0000007_prefix_19107_DEL_r00']) - for sv in svs: - sv.saved_variant_json['genomeVersion'] = '38' - sv.save() - # Test success self.mock_send_slack.reset_mock() self.mock_email.reset_mock() @@ -705,7 +711,7 @@ def test_command(self): set(Family.objects.filter(guid__in=['F000001_1', 'F000003_3']).values_list('analysis_status', flat=True)), {'F'}, ) - self.assertEqual(Family.objects.get(guid='F000014_14').analysis_status, 'Rncc') + self.assertEqual(Family.objects.get(guid='F000014_14').analysis_status, 'S_ng') saved_variant = SavedVariant.objects.get(key=100, family_id=14) self.assertDictEqual(saved_variant.genotypes, {'I000018_na21234': { @@ -731,9 +737,9 @@ def test_command(self): # Test reloading shared annotations is skipped if too many saved variants snv_indel_datasets.delete() - airtable_logs = self.AIRTABLE_LOGS[:-1] + airtable_logs = [] if self.AIRTABLE_LOGS: - airtable_logs.append(('Fetched 1 AnVIL Seqr Loading Requests Tracking records from airtable', None)) + airtable_logs = self.AIRTABLE_LOGS + [('Fetched 1 AnVIL Seqr Loading Requests Tracking records from airtable', None)] self._test_call(num_runs=2, run_loading_logs={ 'GRCh38/SNV_INDEL': [ ('Loading 4 WES SNV_INDEL samples in 2 projects', None), @@ -744,7 +750,7 @@ def test_command(self): class LocalCheckNewSamplesTest(AuthenticationTestCase, CheckNewSamplesTest): - fixtures = ['users', '1kg_project', 'clickhouse_saved_variants'] + fixtures = ['users', '1kg_project', 'clickhouse_discovery_variants'] databases = '__all__' MOCK_DATA_DIR = '/seqr/seqr-hail-search-data' @@ -767,6 +773,7 @@ class LocalCheckNewSamplesTest(AuthenticationTestCase, CheckNewSamplesTest): *Run ID:* manual__2025-01-14 *Validation Errors:* ['Missing the following expected contigs:chr17']""") SLACK_VALIDATION_MESSAGE = '' + SKIPPED_PDO_MESSAGE = '' def setUp(self): patcher = mock.patch('seqr.views.utils.export_utils.os.makedirs') @@ -815,7 +822,7 @@ def _assert_expected_airtable_calls(self, *args, **kwargs): class AirtableCheckNewSamplesTest(AnvilAuthenticationTestCase, CheckNewSamplesTest): - fixtures = ['users', '1kg_project', 'clickhouse_saved_variants'] + fixtures = ['users', '1kg_project', 'clickhouse_discovery_variants'] airtable_samples_url = 'http://testairtable/app3Y97xtbbaOopVR/Samples' airtable_pdo_url = 'http://testairtable/app3Y97xtbbaOopVR/PDO' @@ -841,7 +848,6 @@ class AirtableCheckNewSamplesTest(AnvilAuthenticationTestCase, CheckNewSamplesTe ('Fetching PDO records 0-1 from airtable', None), ('Fetched 1 PDO records from airtable', None), ('Fetching AnVIL Seqr Loading Requests Tracking records 0-2 from airtable', None), - ('Fetched 2 AnVIL Seqr Loading Requests Tracking records from airtable', None), ] VALIDATION_LOGS = [ '==> gsutil ls gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-14/validation_errors.json', @@ -878,6 +884,7 @@ class AirtableCheckNewSamplesTest(AnvilAuthenticationTestCase, CheckNewSamplesTe - Missing the following expected contigs:chr17 The following users have been notified: test_user_manager@test.com""") SLACK_VALIDATION_MESSAGE = '\nSee more at https://storage.cloud.google.com/seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-24/validation_errors.json' + SKIPPED_PDO_MESSAGE = '\n\nSkipped samples in this project have been moved to PDO-1234_sr' def setUp(self): patcher = mock.patch('seqr.utils.file_utils.subprocess.Popen') @@ -891,14 +898,6 @@ def setUp(self): super().setUp() def _add_responses(self): - responses.add( - responses.GET, - self.airtable_loading_tracking_url + self.AIRTABLE_LOADING_QUERY_TEMPLATE.format(EXTERNAL_PROJECT_GUID), - json={'records': [{'id': 'rec12345', 'fields': {}}, {'id': 'rec67890', 'fields': {}}]}) - responses.add( - responses.GET, - self.airtable_loading_tracking_url + self.AIRTABLE_LOADING_QUERY_TEMPLATE.format(EXTERNAL_PROJECT_GUID), - json={'records': [{'id': 'rec12345', 'fields': {}}]}) responses.add( responses.GET, self.airtable_loading_tracking_url + self.AIRTABLE_LOADING_QUERY_TEMPLATE.format('R0002_empty'), @@ -923,6 +922,14 @@ def _add_responses(self): @responses.activate def test_command(self, *args, **kwargs): + responses.add( + responses.GET, + self.airtable_loading_tracking_url + self.AIRTABLE_LOADING_QUERY_TEMPLATE.format(EXTERNAL_PROJECT_GUID), + json={'records': [{'id': 'rec12345', 'fields': {}}, {'id': 'rec67890', 'fields': {}}]}) + responses.add( + responses.GET, + self.airtable_loading_tracking_url + self.AIRTABLE_LOADING_QUERY_TEMPLATE.format(EXTERNAL_PROJECT_GUID), + json={'records': [{'id': 'rec12345', 'fields': {}}]}) self._add_responses() super().test_command(*args, **kwargs) @@ -1042,4 +1049,4 @@ def test_loading_delay_command(self): self._test_success_call(self._anvil_email_calls( email_text=ANVIL_ERROR_TEXT_EMAIL_TEMPLATE.format(error='\n'+ANVIL_ERROR_DELAY), email_html=ANVIL_ERROR_HTML_EMAIL_TEMPLATE.format(error='
'+ANVIL_ERROR_DELAY), - ), next_dataset_id=161) \ No newline at end of file + ), next_dataset_id=161, fetched_tracking=False) \ No newline at end of file diff --git a/seqr/management/tests/reload_saved_variant_genotypes_tests.py b/seqr/management/tests/reload_saved_variant_genotypes_tests.py index 50ec2d6003..272f9a8cac 100644 --- a/seqr/management/tests/reload_saved_variant_genotypes_tests.py +++ b/seqr/management/tests/reload_saved_variant_genotypes_tests.py @@ -5,7 +5,7 @@ class ReloadSavedVariantGenotypesTest(AnvilAuthenticationTestCase): - fixtures = ['users', '1kg_project', 'report_variants', 'clickhouse_saved_variants'] + fixtures = ['users', '1kg_project', 'report_variants', 'clickhouse_saved_variants', 'clickhouse_discovery_variants'] def test_command(self): # Update fixture data diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index 34c77cf5e1..90c13c8e65 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -72,28 +72,28 @@ def test_command(self, mock_datetime, mock_slack, mock_email): }) new_saved_variants = SavedVariant.objects.filter(key__in=[2, 3, 4, 18, 19]).order_by('key').values( - 'key', 'variant_id', 'family_id', 'dataset_type', 'xpos', 'xpos_end', 'ref', 'alt', 'gene_ids', 'genotypes', 'saved_variant_json', 'main_transcript', 'sv_type', + 'key', 'variant_id', 'family_id', 'dataset_type', 'xpos', 'xpos_end', 'ref', 'alt', 'gene_ids', 'genotypes', 'main_transcript', 'sv_type', ) self.maxDiff = None self.assertListEqual(list(new_saved_variants), [{ 'key': 2, 'variant_id': '1-38724419-T-G', 'family_id': 2, 'dataset_type': 'SNV_INDEL', 'xpos': 1038724419, 'xpos_end': 1038724419, 'ref': 'T', 'alt': 'G', 'gene_ids': ['ENSG00000177000', 'ENSG00000277258'], - 'genotypes': VARIANT2['genotypes'], 'saved_variant_json': {}, 'sv_type': None, + 'genotypes': VARIANT2['genotypes'], 'sv_type': None, 'main_transcript': VARIANT2['transcripts']['ENSG00000177000'][0], }, {'key': 3, 'variant_id': '1-91502721-G-A', 'family_id': 2, 'dataset_type': 'SNV_INDEL', 'xpos': 1091502721, 'xpos_end': 1091502721, 'ref': 'G', 'alt': 'A', 'gene_ids': ['ENSG00000097046', 'ENSG00000177000'], - 'genotypes': VARIANT3['genotypes'], 'saved_variant_json': {}, 'sv_type': None, + 'genotypes': VARIANT3['genotypes'], 'sv_type': None, 'main_transcript': VARIANT3['transcripts']['ENSG00000097046'][0], }, {'key': 4, 'variant_id': '1-91511686-T-G', 'family_id': 2, 'dataset_type': 'SNV_INDEL', 'xpos': 1091511686, 'xpos_end': 1091511686, 'ref': 'T', 'alt': 'G', 'gene_ids': ['ENSG00000097046'], - 'genotypes': VARIANT4['genotypes'], 'saved_variant_json': {}, 'sv_type': None, + 'genotypes': VARIANT4['genotypes'], 'sv_type': None, 'main_transcript': VARIANT4['transcripts']['ENSG00000097046'][0], }, {'key': 18, 'variant_id': 'suffix_140593_DUP', 'family_id': 2, 'dataset_type': 'SV_WES', 'xpos': 17038717327, 'xpos_end': 17038719636, 'ref': None, 'alt': None, 'gene_ids': ['ENSG00000275023'], - 'genotypes': GCNV_VARIANT3['genotypes'], 'saved_variant_json': {}, 'main_transcript': {}, 'sv_type': 'DUP', + 'genotypes': GCNV_VARIANT3['genotypes'], 'main_transcript': {}, 'sv_type': 'DUP', }, {'key': 19, 'variant_id': 'suffix_140608_DUP', 'family_id': 2, 'dataset_type': 'SV_WES', 'xpos': 17038721781, 'xpos_end': 17038735703, 'ref': None, 'alt': None, 'gene_ids': ['ENSG00000275023', 'ENSG00000277258', 'ENSG00000277972'], - 'genotypes': GCNV_VARIANT4['genotypes'], 'saved_variant_json': {}, 'main_transcript': {}, 'sv_type': 'DEL', + 'genotypes': GCNV_VARIANT4['genotypes'], 'main_transcript': {}, 'sv_type': 'DEL', }]) expected_tags = { diff --git a/seqr/management/tests/update_individuals_sample_qc_tests.py b/seqr/management/tests/update_individuals_sample_qc_tests.py index 389bf402f1..4c6e0e0562 100644 --- a/seqr/management/tests/update_individuals_sample_qc_tests.py +++ b/seqr/management/tests/update_individuals_sample_qc_tests.py @@ -13,9 +13,12 @@ 'family_samples': { 'F000011_11': ['NA20885'], 'F000012_12': ['NA20888', 'NA20889'], - 'F000014_14': ['NA21234'], }, - 'failed_family_samples': {}, + 'failed_family_samples': { + 'relatedness_check': { + 'F000014_14': {'reasons': ['NA21234 failed']}, + }, + }, 'relatedness_check_file_path': '', 'sample_qc': { 'NA20885': { diff --git a/seqr/migrations/0089_remove_savedvariant_saved_variant_json.py b/seqr/migrations/0089_remove_savedvariant_saved_variant_json.py new file mode 100644 index 0000000000..b2f8d21b6d --- /dev/null +++ b/seqr/migrations/0089_remove_savedvariant_saved_variant_json.py @@ -0,0 +1,93 @@ +# Generated by Django 4.2.24 on 2026-05-04 18:13 + +from django.db import migrations +from pyliftover.liftover import LiftOver + +from seqr.utils.xpos_utils import get_xpos, get_chrom_pos + + +def populate_saved_variant_json_fields(apps, schema_editor): + SavedVariant = apps.get_model('seqr', 'SavedVariant') + db_alias = schema_editor.connection.alias + + saved_variants_q = SavedVariant.objects.using(db_alias) + update_variants = saved_variants_q.filter(key__isnull=True, sv_type__isnull=True) + + # Some edge cases were missed in the initial migration to set the main transcript + to_update = [] + transcript_variants = update_variants.filter( + gene_ids__len__gt=0, main_transcript={}, saved_variant_json__mainTranscriptId__isnull=False, + ) + for variant in transcript_variants: + main_transcript_id = variant.selected_main_transcript_id or variant.saved_variant_json.get('mainTranscriptId') + if main_transcript_id: + variant.main_transcript = next( + t for ts in variant.saved_variant_json['transcripts'].values() for t in ts + if t['transcriptId'] == main_transcript_id + ) + to_update.append(variant) + if to_update: + updated = saved_variants_q.bulk_update(to_update, ['main_transcript']) + print(f'Updated main transcript for {updated} variants') + + # Some variants failed to lift to 38 when project build was updated, run liftover on these + to_update = [] + lifover_variants = update_variants.filter(saved_variant_json__genomeVersion='37', family__project__genome_version='38') + if lifover_variants.exists(): + VariantNote = apps.get_model('seqr', 'VariantNote') + note_q = VariantNote.objects.using(db_alias) + _liftover_variants(lifover_variants, saved_variants_q, note_q) + + # Some variants are inexplicably saved on 38 despite being in 37 projects, + # use existing liftover to map to the correct project genome version + to_update = [] + liftunder_variants = update_variants.filter(saved_variant_json__genomeVersion='38', family__project__genome_version='37') + for variant in liftunder_variants: + chrom = variant.saved_variant_json['liftedOverChrom'] + pos = variant.saved_variant_json['liftedOverPos'] + variant.xpos = get_xpos(chrom, pos) + variant.variant_id = f'{chrom}-{pos}-{variant.ref}-{variant.alt}' + to_update.append(variant) + if to_update: + updated = saved_variants_q.bulk_update(to_update, ['xpos', 'variant_id']) + print(f'Lifted under {updated} variants') + + +def _liftover_variants(lifover_variants, saved_variants_q, note_q): + to_update = [] + skipped = 0 + liftover = LiftOver('hg19', 'hg38') + for variant in lifover_variants: + chrom, pos = get_chrom_pos(variant.xpos) + lifted_coord = liftover.convert_coordinate(f'chr{chrom}', pos) + + if not lifted_coord: + note = note_q.create(note='Variant tagged on build GRCh37', guid=variant.guid.replace('SV', 'VN')) + note.saved_variants.add(variant) + skipped += 1 + continue + + lifted_pos = lifted_coord[0][1] + variant.xpos = get_xpos(chrom, lifted_pos) + variant.variant_id = f'{chrom}-{lifted_pos}-{variant.ref}-{variant.alt}' + to_update.append(variant) + + updated = saved_variants_q.bulk_update(to_update, ['xpos', 'variant_id']) + print(f'Lifted over {updated} variants') + if skipped: + print(f'Skipped liftover for {skipped} variants, added notes to affected variants') + + +class Migration(migrations.Migration): + + dependencies = [ + ('seqr', '0088_rnasample_sequencing_type'), + ] + + operations = [ + migrations.RunPython(populate_saved_variant_json_fields, reverse_code=migrations.RunPython.noop), + migrations.RemoveField( + model_name='savedvariant', + name='saved_variant_json', + ), + ] diff --git a/seqr/models.py b/seqr/models.py index 149dd30655..20b6d1ca32 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -817,7 +817,6 @@ class SavedVariant(ModelWithGUID): key = models.PositiveBigIntegerField(null=True, blank=True) selected_main_transcript_id = models.CharField(max_length=20, null=True) - saved_variant_json = models.JSONField(default=dict) genotypes = models.JSONField(default=dict) main_transcript = models.JSONField(default=dict, encoder=DjangoJSONEncoderWithSets) gene_ids = ArrayField(models.CharField(max_length=20), null=True, blank=True) @@ -1150,9 +1149,7 @@ def bulk_create(cls, user, new_models, **kwargs): return models @classmethod - def bulk_delete(cls, user, queryset=None, **filter_kwargs): - if queryset is None: - queryset = cls.objects.filter(**filter_kwargs) + def bulk_delete(cls, user, queryset): cls.log_model_no_guid_bulk_update(queryset, user, 'delete') return queryset.delete() diff --git a/seqr/utils/social_auth_pipeline_tests.py b/seqr/utils/social_auth_pipeline_tests.py index 9208004d85..fba3e39be0 100644 --- a/seqr/utils/social_auth_pipeline_tests.py +++ b/seqr/utils/social_auth_pipeline_tests.py @@ -1,67 +1,65 @@ import mock import responses -from unittest import TestCase from social_core.backends.google import GoogleOAuth2 from seqr.utils.social_auth_pipeline import validate_anvil_registration, validate_user_exist, log_signed_in -from seqr.views.utils.test_utils import TEST_TERRA_API_ROOT_URL, REGISTER_RESPONSE +from seqr.views.utils.test_utils import AuthenticationTestCase, TEST_TERRA_API_ROOT_URL, REGISTER_RESPONSE @mock.patch('seqr.views.utils.terra_api_utils.TERRA_API_ROOT_URL', TEST_TERRA_API_ROOT_URL) -class SocialAuthPipelineTest(TestCase): +class SocialAuthPipelineTest(AuthenticationTestCase): + + fixtures = ['users'] @responses.activate - @mock.patch('seqr.utils.social_auth_pipeline.logger') - def test_validate_anvil_registration(self, mock_logger): + def test_validate_anvil_registration(self): url = TEST_TERRA_API_ROOT_URL + 'register' responses.add(responses.GET, url, status=404) r = validate_anvil_registration(GoogleOAuth2(), {'access_token': '', 'email': 'test@seqr.org'}) # nosec - mock_logger.warning.assert_called_with( - 'User test@seqr.org is trying to login without registration on AnVIL. None called Terra API: GET /register got status 404 with reason: Not Found', - extra={'user_email': 'test@seqr.org'} - ) + self.assert_json_logs(None, [ + ('User test@seqr.org is trying to login without registration on AnVIL. None called Terra API: GET /register got status 404 with reason: Not Found', { + 'severity': 'WARNING', 'user': 'test@seqr.org', + }) + ]) self.assertEqual(r.url, '/login/error/anvil_registration') - self.assertEqual(len(mock_logger.method_calls), 1) backend = GoogleOAuth2() backend.strategy.session_set('next', '/foo/bar') r = validate_anvil_registration(backend, {'access_token': '', 'email': 'test@seqr.org'}) # nosec self.assertEqual(r.url, '/login/error/anvil_registration?next=%2Ffoo%2Fbar') - mock_logger.reset_mock() + self.reset_logs() responses.replace(responses.GET, url, status=200, body=REGISTER_RESPONSE) r = validate_anvil_registration(GoogleOAuth2(), {'access_token': '', 'email': 'test@seqr.org'}) - mock_logger.warning.assert_not_called() + self.assert_json_logs(None, [(f'GET {url} 200 127', None)]) self.assertIsNone(r) - @mock.patch('seqr.utils.social_auth_pipeline.logger') - def test_validate_user_exist(self, mock_logger): + def test_validate_user_exist(self): validate_user_exist(GoogleOAuth2(), {'email': 'test_user_manager@test.com'}, user='test') - self.assertEqual(len(mock_logger.method_calls), 0) + self.assert_no_logs() r = validate_user_exist(GoogleOAuth2(), {'email': 'test_user_manager@test.com'}) - mock_logger.warning.assert_called_with( - 'Google user test_user_manager@test.com is trying to login without an existing seqr account (google-oauth2).', - extra={'user_email': 'test_user_manager@test.com'}) + self.assert_json_logs(None, [ + ('Google user test_user_manager@test.com is trying to login without an existing seqr account (google-oauth2).', { + 'severity': 'WARNING', 'user': 'test_user_manager@test.com', + }) + ]) self.assertEqual(r.url, '/login/error/no_account') - self.assertEqual(len(mock_logger.method_calls), 1) backend = GoogleOAuth2() backend.strategy.session_set('next', '/foo/bar') r = validate_user_exist(backend, {'email': 'test_user_manager@test.com'}) self.assertEqual(r.url, '/login/error/no_account?next=%2Ffoo%2Fbar') - @mock.patch('seqr.utils.social_auth_pipeline.logger') - def test_log_signed_in(self, mock_logger): + def test_log_signed_in(self): log_signed_in(GoogleOAuth2(), {'email': 'test_user_manager@test.com'}, user='test') - mock_logger.info.assert_called_with('Logged in test_user_manager@test.com (google-oauth2)', - extra={'user_email': 'test_user_manager@test.com'}) - self.assertEqual(len(mock_logger.method_calls), 1) + self.assert_json_logs(None, [ + ('Logged in test_user_manager@test.com (google-oauth2)', {'user': 'test_user_manager@test.com'}) + ]) - mock_logger.reset_mock() + self.reset_logs() log_signed_in(GoogleOAuth2(), {'email': 'test_user_manager@test.com'}, is_new=True, user='test') - mock_logger.info.assert_has_calls([ - mock.call('Logged in test_user_manager@test.com (google-oauth2)', extra={'user_email': 'test_user_manager@test.com'}), - mock.call('Created user test_user_manager@test.com (google-oauth2)', extra={'user_email': 'test_user_manager@test.com'}), + self.assert_json_logs(None, [ + ('Logged in test_user_manager@test.com (google-oauth2)', {'user': 'test_user_manager@test.com'}), + ('Created user test_user_manager@test.com (google-oauth2)', {'user': 'test_user_manager@test.com'}) ]) - self.assertEqual(len(mock_logger.method_calls), 2) diff --git a/seqr/views/apis/anvil_workspace_api_tests.py b/seqr/views/apis/anvil_workspace_api_tests.py index 70266db8a4..2c444a7c6f 100644 --- a/seqr/views/apis/anvil_workspace_api_tests.py +++ b/seqr/views/apis/anvil_workspace_api_tests.py @@ -256,6 +256,10 @@ def test_grant_workspace_access(self, mock_add_service_account, mock_has_service mock_has_service_account.return_value = False response = self.client.post(url, content_type='application/json', data=json.dumps(GRANT_ACCESS_BODY)) self.assertEqual(response.status_code, 400) + mock_logger.info.assert_called_with( + f'Added service account for {TEST_WORKSPACE_NAMESPACE}/{TEST_NO_PROJECT_WORKSPACE_NAME}, waiting for access to grant', + self.manager_user, + ) self.assertEqual(response.json()['error'], 'Failed to grant seqr service account access to the workspace') mock_has_service_account.assert_called_with(self.manager_user, TEST_WORKSPACE_NAMESPACE, TEST_NO_PROJECT_WORKSPACE_NAME) @@ -264,6 +268,7 @@ def test_grant_workspace_access(self, mock_add_service_account, mock_has_service # Test valid operation mock_time.reset_mock() + mock_logger.reset_mock() mock_has_service_account.reset_mock() mock_add_service_account.return_value = False response = self.client.post(url, content_type='application/json', data=json.dumps(GRANT_ACCESS_BODY)) @@ -273,6 +278,17 @@ def test_grant_workspace_access(self, mock_add_service_account, mock_has_service TEST_NO_PROJECT_WORKSPACE_NAME) mock_has_service_account.assert_not_called() mock_time.sleep.assert_not_called() + mock_logger.assert_not_called() + + mock_time.reset_mock() + mock_add_service_account.return_value = True + mock_has_service_account.return_value = True + response = self.client.post(url, content_type='application/json', data=json.dumps(GRANT_ACCESS_BODY)) + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), {'success': True}) + mock_add_service_account.assert_called_with(self.manager_user, TEST_WORKSPACE_NAMESPACE, TEST_NO_PROJECT_WORKSPACE_NAME) + mock_has_service_account.assert_called_with(self.manager_user, TEST_WORKSPACE_NAMESPACE, TEST_NO_PROJECT_WORKSPACE_NAME) + mock_time.sleep.assert_called_with(3) mock_logger.info.assert_called_with( f'Added service account for {TEST_WORKSPACE_NAMESPACE}/{TEST_NO_PROJECT_WORKSPACE_NAME}, waiting for access to grant', self.manager_user, diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index b40b2a37d6..e5b397a934 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -898,6 +898,7 @@ def test_load_phenotype_prioritization_data(self, mock_subprocess, mock_send_ema # Test uploading new data self.reset_logs() mock_send_email.reset_mock() + mock_send_email.side_effect = Exception('Email server down') mock_subprocess.return_value.stdout = self._join_data(PHENOTYPE_PRIORITIZATION_HEADER + UPDATE_LIRICAL_DATA) mock_random.randint.side_effect = [177442291, 215071655] response = self.client.post(url, content_type='application/json', data=json.dumps(request_body)) @@ -919,33 +920,44 @@ def test_load_phenotype_prioritization_data(self, mock_subprocess, mock_send_ema 'entityIds': ['PP177442291_na19678ensg0000010', 'PP215071655_na19678ensg0000010'], }}), ]) + email_body = 'data for 2 Lirical sample(s)' + self.assert_json_logs(user=None, offset=4, expected=[ + ('Error sending project email for R0001_1kg: Email server down', { + 'severity': 'ERROR', + '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', + 'detail': { + 'to': ['test_user_manager@test.com'], + 'subject': 'New Lirical data available in seqr', + 'email_body': self._expected_email_body(email_body=email_body), + }, + }), + ]) saved_data = _get_json_for_models(PhenotypePrioritization.objects.filter(tool='lirical').order_by('id'), nested_fields=[{'fields': ('individual', 'guid'), 'key': 'individualGuid'}]) self.assertListEqual(saved_data, EXPECTED_UPDATED_LIRICAL_DATA) self._assert_expected_notifications(mock_send_email, [ - {'data_type': 'Lirical', 'user': self.data_manager_user, 'email_body': 'data for 2 Lirical sample(s)'}, + {'data_type': 'Lirical', 'user': self.data_manager_user, 'email_body': email_body}, ]) @staticmethod - def _assert_expected_notifications(mock_send_email, expected_notifs: list[dict]): - calls = [] - for notif_dict in expected_notifs: - project_guid = notif_dict.get('project_guid', PROJECT_GUID) - project_name = notif_dict.get('project_name', '1kg project nåme with uniçøde') - url = f'https://test-seqr.org/project/{project_guid}/project_page' - project_link = f'{project_name}' - expected_email_body = ( - f'Dear seqr user,\n\nThis is to notify you that {notif_dict["email_body"]} ' - f'has been loaded in seqr project {project_link}\n\nAll the best,\nThe seqr team' - ) - calls.append( - mock.call( - email_body=expected_email_body, - subject=f'New {notif_dict["data_type"]} data available in seqr', - to=['test_user_manager@test.com'], - process_message=_set_bulk_notification_stream, - ) - ) + def _expected_email_body(email_body='', project_guid=PROJECT_GUID, project_name='1kg project nåme with uniçøde', **kwargs): + url = f'https://test-seqr.org/project/{project_guid}/project_page' + project_link = f'{project_name}' + return ( + f'Dear seqr user,\n\nThis is to notify you that {email_body} ' + f'has been loaded in seqr project {project_link}\n\nAll the best,\nThe seqr team' + ) + + @classmethod + def _assert_expected_notifications(cls, mock_send_email, expected_notifs: list[dict]): + calls = [ + mock.call( + email_body=cls._expected_email_body(**notif_dict), + subject=f'New {notif_dict["data_type"]} data available in seqr', + to=['test_user_manager@test.com'], + process_message=_set_bulk_notification_stream, + ) for notif_dict in expected_notifs + ] mock_send_email.assert_has_calls(calls) def test_loading_vcfs(self): @@ -1036,6 +1048,12 @@ def test_validate_callset(self): response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 200) + self._add_file_iter([]) + response = self.client.post(url, content_type='application/json', data=json.dumps({ + **self.REQUEST_BODY, 'filePath': f'{self.CALLSET_DIR}/mito_calls.mt', 'datasetType': 'MITO', + })) + self._assert_expected_validate_mito_response(response) + @mock.patch('seqr.views.utils.permissions_utils.INTERNAL_NAMESPACES', ['my-seqr-billing', 'ext-data']) @mock.patch('seqr.views.utils.airtable_utils.BASE_URL', 'https://seqr.broadinstitute.org/') @responses.activate @@ -1428,6 +1446,13 @@ def _test_update_rna_seq(self, data_type, *args, **kwargs): self.assertEqual(response.status_code, 500) self.assertDictEqual(response.json(), {'error': 'Airtable is not configured'}) + def _assert_expected_validate_mito_response(self, response): + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), { + 'errors': ['Invalid VCF file format - file path must end with .vcf or .vcf.gz or .vcf.bgz'], + 'warnings': None, + }) + @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') class AnvilDataManagerAPITest(AnvilAuthenticationTestCase, DataManagerAPITest): @@ -1626,18 +1651,22 @@ def _trigger_error(self, url, body, variables, mock_open, mock_gzip_open, mock_m required_sample_field='gCNV_CallsetPath', additional_vcf_ids=",SeqrIDWithMismatch='NA21987'", ) + responses.replace(responses.GET, 'https://api.airtable.com/v0/app3Y97xtbbaOopVR/Samples', json=INVALID_AIRTABLE_SAMPLE_RECORDS, status=200) responses.calls.reset() response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), { 'warnings': None, 'errors': [ + 'The following samples are associated with misconfigured PDOs in Airtable: HG00731, NA21234', 'The following families have previously loaded samples absent from airtable\nFamily fam14: NA21234, NA21654', + 'The following samples are associated with misconfigured PDOs in Airtable: HG00731, NA21234', 'The following samples are included in airtable but are missing from the VCF: NA21987', ], }) self.assertEqual(len(responses.calls), 2) self._assert_expected_airtable_call(required_sample_field='SV_CallsetPath', project_guid='R0004_non_analyst_project') + responses.replace(responses.GET, 'https://api.airtable.com/v0/app3Y97xtbbaOopVR/Samples', json=AIRTABLE_SAMPLE_RECORDS, status=200) def _test_load_single_project(self, mock_open, mock_gzip_open, mock_mkdir, response, *args, url=None, body=None, **kwargs): super()._test_load_single_project(mock_open, mock_gzip_open, mock_mkdir, response, url, body) @@ -1741,3 +1770,7 @@ def _assert_expected_airtable_errors(self, url): self.assertDictEqual(response.json(), { 'error': 'The following samples are associated with misconfigured PDOs in Airtable: HG00731, NA21234', }) + + def _assert_expected_validate_mito_response(self, response): + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), {'vcfSamples': None}) diff --git a/seqr/views/apis/individual_api.py b/seqr/views/apis/individual_api.py index 75846a1bb1..06da8a78a8 100644 --- a/seqr/views/apis/individual_api.py +++ b/seqr/views/apis/individual_api.py @@ -359,8 +359,6 @@ def _update_and_parse_individuals_and_families(project, individual_records, user AR_DSPERM_COL = 'ar_donorsperm' def _bool_value(val): - if isinstance(val, bool): - return val if val.lower() == 'true': return True elif val.lower() == 'false': @@ -368,8 +366,6 @@ def _bool_value(val): raise ValueError def _array_value(val): - if isinstance(val, list): - return val return [o.strip() for o in val.split(',')] def _gene_value(val): @@ -380,8 +376,6 @@ def _gene_value(val): return gene def _gene_list_value(val): - if isinstance(val, list): - return val seperator_escaped_val = ''.join(m.replace(',', ';') if not m.startswith('(') else m for m in re.split('(\([^)]+\))', val)) return [_gene_value(o) for o in seperator_escaped_val.split(';')] @@ -410,60 +404,6 @@ def _gene_list_value(val): CANDIDATE_GENES_COL: _gene_list_value, } -def _get_year(val): - return datetime.strptime(val, '%Y-%m-%d').year - -def _nested_val(nested_key): - return lambda val: val.get(nested_key) - -def _get_phenotips_features(observed): - def get_observed_features(features): - return [{'id': feature['id']} for feature in features if feature['observed'] == observed] - return get_observed_features - -PHENOTIPS_JSON_FIELD_MAP = { - 'family_id': [(FAMILY_ID_COL, None)], - 'external_id': [(INDIVIDUAL_ID_COL, None)], - 'features': [ - (FEATURES_COL, _get_phenotips_features('yes')), - (ABSENT_FEATURES_COL, _get_phenotips_features('no')), - ], - 'date_of_birth': [(BIRTH_COL, _get_year)], - 'date_of_death': [(DEATH_COL, _get_year)], - 'global_age_of_onset': [(ONSET_AGE_COL, lambda val: val[0]['label'])], - 'family_history': [ - (CONSANGUINITY_COL, _nested_val('consanguinity')), - (AFFECTED_REL_COL, _nested_val('affectedRelatives')), - ], - 'global_mode_of_inheritance': [(EXP_INHERITANCE_COL, lambda val: [o['label'] for o in val])], - 'prenatal_perinatal_history': [ - (AR_FM_COL, _nested_val('assistedReproduction_fertilityMeds')), - (AR_IUI_COL, _nested_val('assistedReproduction_iui')), - (AR_IVF_COL, _nested_val('ivf')), - (AR_ICSI_COL, _nested_val('icsi')), - (AR_SURROGACY_COL, _nested_val('assistedReproduction_surrogacy')), - (AR_DEGG_COL, _nested_val('assistedReproduction_donoregg')), - (AR_DSPERM_COL, _nested_val('assistedReproduction_donorsperm')), - ], - 'ethnicity': [ - (MAT_ETHNICITY_COL, _nested_val('maternal_ethnicity')), - (PAT_ETHNICITY_COL, _nested_val('paternal_ethnicity')), - ], - 'disorders': [(DISORDERS_COL, lambda val: [int(d['id'].lstrip('MIM:')) for d in val])], - 'genes': [(CANDIDATE_GENES_COL, None)], - 'rejectedGenes': [(REJECTED_GENES_COL, None)], -} - -def _parse_phenotips_record(row): - record = {} - for k, formatters in PHENOTIPS_JSON_FIELD_MAP.items(): - val = row.get(k) - if val: - for col, formatter in formatters: - field_val = formatter(val) if formatter else val - if field_val is not None: - record[col] = field_val - return record @login_and_policies_required def receive_individuals_metadata_handler(request, project_guid): @@ -500,64 +440,61 @@ def process_records(json_records, filename=''): def _process_hpo_records(records, filename, project, user): - if filename.endswith('.json'): - row_dicts = [_parse_phenotips_record(record) for record in records] - else: - column_map = {} - for i, field in enumerate(records[0]): - key = field.lower() - if re.match("hpo.*present", key): - column_map[FEATURES_COL] = i - elif re.match("hpo.*absent", key): - column_map[ABSENT_FEATURES_COL] = i - elif re.match("hp.*number*", key): - if not HPO_TERM_NUMBER_COL in column_map: - column_map[HPO_TERM_NUMBER_COL] = [] - column_map[HPO_TERM_NUMBER_COL].append(i) - elif 'family' in key or 'pedigree' in key: - column_map[FAMILY_ID_COL] = i + column_map = {} + for i, field in enumerate(records[0]): + key = field.lower() + if re.match("hpo.*present", key): + column_map[FEATURES_COL] = i + elif re.match("hpo.*absent", key): + column_map[ABSENT_FEATURES_COL] = i + elif re.match("hp.*number*", key): + if not HPO_TERM_NUMBER_COL in column_map: + column_map[HPO_TERM_NUMBER_COL] = [] + column_map[HPO_TERM_NUMBER_COL].append(i) + elif 'family' in key or 'pedigree' in key: + column_map[FAMILY_ID_COL] = i + else: + col_key = next((col for col, text in [ + (NOTES_COL, 'notes'), (INDIVIDUAL_ID_COL, 'individual'), (AFFECTED_REL_COL, 'affected relative'), + (AFFECTED_FEATURE_COL, 'affected'), (BIRTH_COL, 'birth'), (DEATH_COL, 'death'), + (ONSET_AGE_COL, 'onset'), (AR_ICSI_COL, 'relative'), + (CONSANGUINITY_COL, 'consanguinity'), (EXP_INHERITANCE_COL, 'inheritance'), (AR_FM_COL, 'fertility'), + (AR_IUI_COL, 'intrauterine'), (AR_IVF_COL, 'in vitro'), (AR_ICSI_COL, 'cytoplasmic'), + (AR_SURROGACY_COL, 'surrogacy'), (AR_DEGG_COL, 'donor egg'), (AR_DSPERM_COL, 'donor sperm'), + (MAT_ETHNICITY_COL, 'maternal ancestry'), (PAT_ETHNICITY_COL, 'paternal ancestry'), + (DISORDERS_COL, 'disorders'), (REJECTED_GENES_COL, 'tested genes'), + (CANDIDATE_GENES_COL, 'candidate genes'), (ASSIGNED_ANALYST_COL, 'assigned analyst'), + ] if text in key), None) + if col_key: + column_map[col_key] = i + + if INDIVIDUAL_ID_COL not in column_map: + raise ValueError('Invalid header, missing individual id column') + + row_dicts = [{column: row[index] if isinstance(index, int) else next((row[i] for i in index if row[i]), None) + for column, index in column_map.items()} for row in records[1:]] + + if FEATURES_COL in column_map or ABSENT_FEATURES_COL in column_map: + for row in row_dicts: + row[FEATURES_COL] = parse_hpo_terms(row.get(FEATURES_COL)) + row[ABSENT_FEATURES_COL] = parse_hpo_terms(row.get(ABSENT_FEATURES_COL)) + + elif HPO_TERM_NUMBER_COL in column_map: + aggregate_rows = defaultdict(lambda: {FEATURES_COL: set(), ABSENT_FEATURES_COL: set()}) + for row in row_dicts: + column = ABSENT_FEATURES_COL if row.pop(AFFECTED_FEATURE_COL) == 'no' else FEATURES_COL + aggregate_entry = aggregate_rows[(row.get(FAMILY_ID_COL), row.get(INDIVIDUAL_ID_COL))] + term = row.pop(HPO_TERM_NUMBER_COL, None) + if term: + aggregate_entry[column].add(term.strip()) else: - col_key = next((col for col, text in [ - (NOTES_COL, 'notes'), (INDIVIDUAL_ID_COL, 'individual'), (AFFECTED_REL_COL, 'affected relative'), - (AFFECTED_FEATURE_COL, 'affected'), (BIRTH_COL, 'birth'), (DEATH_COL, 'death'), - (ONSET_AGE_COL, 'onset'), (AR_ICSI_COL, 'relative'), - (CONSANGUINITY_COL, 'consanguinity'), (EXP_INHERITANCE_COL, 'inheritance'), (AR_FM_COL, 'fertility'), - (AR_IUI_COL, 'intrauterine'), (AR_IVF_COL, 'in vitro'), (AR_ICSI_COL, 'cytoplasmic'), - (AR_SURROGACY_COL, 'surrogacy'), (AR_DEGG_COL, 'donor egg'), (AR_DSPERM_COL, 'donor sperm'), - (MAT_ETHNICITY_COL, 'maternal ancestry'), (PAT_ETHNICITY_COL, 'paternal ancestry'), - (DISORDERS_COL, 'disorders'), (REJECTED_GENES_COL, 'tested genes'), - (CANDIDATE_GENES_COL, 'candidate genes'), (ASSIGNED_ANALYST_COL, 'assigned analyst'), - ] if text in key), None) - if col_key: - column_map[col_key] = i - - if INDIVIDUAL_ID_COL not in column_map: - raise ValueError('Invalid header, missing individual id column') - - row_dicts = [{column: row[index] if isinstance(index, int) else next((row[i] for i in index if row[i]), None) - for column, index in column_map.items()} for row in records[1:]] - - if FEATURES_COL in column_map or ABSENT_FEATURES_COL in column_map: - for row in row_dicts: - row[FEATURES_COL] = parse_hpo_terms(row.get(FEATURES_COL)) - row[ABSENT_FEATURES_COL] = parse_hpo_terms(row.get(ABSENT_FEATURES_COL)) - - elif HPO_TERM_NUMBER_COL in column_map: - aggregate_rows = defaultdict(lambda: {FEATURES_COL: set(), ABSENT_FEATURES_COL: set()}) - for row in row_dicts: - column = ABSENT_FEATURES_COL if row.pop(AFFECTED_FEATURE_COL) == 'no' else FEATURES_COL - aggregate_entry = aggregate_rows[(row.get(FAMILY_ID_COL), row.get(INDIVIDUAL_ID_COL))] - term = row.pop(HPO_TERM_NUMBER_COL, None) - if term: - aggregate_entry[column].add(term.strip()) - else: - aggregate_entry[column] = set() - aggregate_entry.update({k: v for k, v in row.items() if v}) - - row_dicts = [ - {**entry, **{col: [{'id': feature} for feature in entry[col]] for col in [FEATURES_COL, ABSENT_FEATURES_COL]}} - for entry in aggregate_rows.values() - ] + aggregate_entry[column] = set() + aggregate_entry.update({k: v for k, v in row.items() if v}) + + row_dicts = [ + {**entry, **{col: [{'id': feature} for feature in entry[col]] for col in [FEATURES_COL, ABSENT_FEATURES_COL]}} + for entry in aggregate_rows.values() + ] return _parse_individual_hpo_terms(row_dicts, project, user) @@ -893,9 +830,7 @@ def _parse_new_aip_saved_variants(new_variant_keys, family_variant_data): if variant_id in variants_by_id: variant.update(variants_by_id[variant_id]) else: - variant.update({'key': None, 'saved_variant_json': {k: v for k, v in variant.items() if k in { - 'chrom', 'pos', 'ref', 'alt', 'variantId', 'xpos', 'genomeVersion', 'genotypes', 'transcripts', 'mainTranscriptId', - }}}) + variant.update({'key': None}) new_variant_data[key] = variant return new_variant_data diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index dc0e8f248b..a68a391116 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -93,6 +93,7 @@ LOAD_PARTICIPANT_TABLE = deepcopy(PARTICIPANT_TABLE) for row in LOAD_PARTICIPANT_TABLE[4:]: row[7] = row[7].replace('Broad_', '') +LOAD_PARTICIPANT_TABLE[5][11] = 'Niece' LOAD_PARTICIPANT_TABLE[6][15] += '|Asian' LOAD_PARTICIPANT_TABLE[6][17] = '' @@ -1019,11 +1020,11 @@ def test_individuals_metadata_table_handler(self): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'errors': ['Invalid header, missing individual id column'], 'warnings': []}) - header = 'family_id,individual_id,hpo_term_present,hpo_term_absent,sex,birth year,other affected relatives,onset age,expected inheritance,maternal ancestry,candidate genes,assigned analyst' + header = 'individual_id,hpo_term_present,hpo_term_absent,sex,birth year,other affected relatives,onset age,expected inheritance,maternal ancestry,candidate genes,assigned analyst' rows = [ - '1,NA19678,,,,,no,infant,recessive,,,not_an_email', - '1,NA19679,HP:0100258 (Preaxial polydactyly),,,,,,,,,test_user_no_access@test.com', - '1,HG00731,HP:0002017,HP:0012469 (Infantile spasms);HP:0011675 (Arrhythmia);HP:0011675 (Arrhythmia),,,,,,,,,', + 'NA19678,,,,,no,infant,recessive,,,not_an_email', + 'NA19679,HP:0100258 (Preaxial polydactyly),,,,,,,,,test_user_no_access@test.com', + 'HG007311,HP:0002017,HP:0012469 (Infantile spasms);HP:0011675 (Arrhythmia);HP:0011675 (Arrhythmia),,,,,,,,,', ] f = SimpleUploadedFile('updates.csv', "{}\n{}".format(header, '\n'.join(rows)).encode('utf-8')) response = self.client.post(url, data={'f': f}) @@ -1038,12 +1039,15 @@ def test_individuals_metadata_table_handler(self): 'The following invalid values for "onset_age" will not be added: infant (NA19678)', 'The following invalid values for "expected_inheritance" will not be added: recessive (NA19678)', 'The following invalid values for "assigned_analyst" will not be added: not_an_email (NA19678); test_user_no_access@test.com (NA19679)', - 'Unable to find matching ids for 1 individuals. The following entries will not be updated: HG00731', + 'Unable to find matching ids for 1 individuals. The following entries will not be updated: HG007311', 'No changes detected for 2 individuals. The following entries will not be updated: NA19678, NA19679', ]}) # send valid request + header = f'family_id,{header}' rows[0] = '1,NA19678,,,,,false,,,,,' + rows[1] = f'1,{rows[1]}' + rows[2] = rows[2].replace('HG007311', '1,HG00731') rows.append('1,NA19675_1,HP:0002017,"HP:0012469 (Infantile spasms);HP:0004322 (Short stature, severe)",F,2000,True,Juvenile onset,"Autosomal dominant inheritance, Sporadic","Finnish, Irish","IKBKAP -- (multiple panels, no confirm), EHBP1L1",test_user_collaborator@test.com') f = SimpleUploadedFile('updates.csv', "{}\n{}".format(header, '\n'.join(rows)).encode('utf-8')) response = self.client.post(url, data={'f': f}) @@ -1072,8 +1076,12 @@ def test_individuals_metadata_hpo_term_number_table_handler(self): def _set_metadata_file_iter(self, genetic_findings_table): self.gs_files.update({ f'{file_name}.tsv': iter(['\t'.join(row).encode() for row in file]) for file_name, file in [ - ('experiment_dna_short_read', EXPERIMENT_TABLE), - ('experiment', EXPERIMENT_LOOKUP_TABLE), + ('experiment_dna_short_read', EXPERIMENT_TABLE + [[ + 'Broad_exome_HG00732', 'Broad_SM-JDBTM', 'Broad_HG00732_1', 'Kapa HyperPrep', '151', 'exome', '', '', '2022-08-15', '385', 'NovaSeq', '', + ]]), + ('experiment', EXPERIMENT_LOOKUP_TABLE + [[ + 'experiment_dna_short_read.Broad_exome_HG00732', 'experiment_dna_short_read', 'Broad_exome_HG00732', 'Broad_HG00732', + ]]), ('participant', LOAD_PARTICIPANT_TABLE), ('phenotype', PHENOTYPE_TABLE), ('genetic_findings', genetic_findings_table), @@ -1121,6 +1129,7 @@ def test_import_gregor_metadata(self): }) warnings = [ 'Broad_HG00733 is the mother of VCGS_FAM203_621_D2 but is not included', + 'Skipped Invalid proband relationship "Niece" for NA20888 with given gender Male', 'Skipped the following unrecognized HPO terms: HP:0001509', ] self.assertDictEqual(response_json['importStats'], {'gregorMetadata': { @@ -1171,8 +1180,8 @@ def test_import_gregor_metadata(self): 'mother__individual_id', 'father__individual_id', 'features', 'absent_features', 'case_review_status', ) self.assertDictEqual(individual_db_data[0], { - 'individual_id': 'Broad_HG00732', - 'display_name': '', + 'individual_id': 'Broad_HG00732_1', + 'display_name': 'Broad_HG00732', 'family__guid': new_family_guid, 'affected': 'N', 'sex': 'M', @@ -1204,7 +1213,7 @@ def test_import_gregor_metadata(self): 'family__guid': 'F000012_12', 'affected': 'A', 'sex': 'M', - 'proband_relationship': '', + 'proband_relationship': None, 'mother__individual_id': None, 'father__individual_id': None, 'population': 'SAS', @@ -1220,7 +1229,7 @@ def test_import_gregor_metadata(self): 'sex': 'F', 'proband_relationship': 'S', 'mother__individual_id': None, - 'father__individual_id': 'Broad_HG00732', + 'father__individual_id': 'Broad_HG00732_1', 'population': 'AMR', 'features': [{'id': 'HP:0011675'}], 'absent_features': [{'id': 'HP:0002017'}], @@ -1230,7 +1239,7 @@ def test_import_gregor_metadata(self): saved_variants = SavedVariant.objects.filter( varianttag__variant_tag_type__name='GREGoR Finding' ).order_by('family_id', 'variant_id').distinct().values( - 'guid', 'variant_id', 'xpos', 'family__guid', 'saved_variant_json', 'key', 'dataset_type', 'genotypes', 'gene_ids', 'main_transcript', + 'guid', 'variant_id', 'xpos', 'family__guid', 'key', 'dataset_type', 'genotypes', 'gene_ids', 'main_transcript', ) self.assertEqual(len(saved_variants), 4) self.assertDictEqual(saved_variants[0], { @@ -1238,7 +1247,6 @@ def test_import_gregor_metadata(self): 'variant_id': '1-248367227-TC-T', 'xpos': 1248367227, 'family__guid': 'F000012_12', - 'saved_variant_json': {}, 'key': 100, 'dataset_type': 'SNV_INDEL', 'genotypes': mock.ANY, @@ -1256,20 +1264,6 @@ def test_import_gregor_metadata(self): 'variant_id': '1-249045487-A-G', 'xpos': 1249045487, 'family__guid': 'F000012_12', - 'saved_variant_json': { - 'alt': 'G', - 'chrom': '1', - 'genomeVersion': '37', - 'genotypes': {created_individual_guid: {'numAlt': 1}}, - 'mainTranscriptId': None, - 'pos': 249045487, - 'ref': 'A', - 'variantId': '1-249045487-A-G', - 'xpos': 1249045487, - 'transcripts': { - 'ENSG00000240361': [{'hgvsc': None, 'hgvsp': None, 'transcriptId': None}], - }, - }, 'main_transcript': {}, 'key': None, 'dataset_type': 'SNV_INDEL', @@ -1282,7 +1276,6 @@ def test_import_gregor_metadata(self): 'variant_id': '1-248367227-TC-T', 'xpos': 1248367227, 'family__guid': new_family_guid, - 'saved_variant_json': {}, 'key': 100, 'dataset_type': 'SNV_INDEL', 'genotypes': new_family_genotypes, @@ -1340,7 +1333,6 @@ def test_import_gregor_metadata(self): saved_variant = SavedVariant.objects.get(family__guid=new_family_guid, variant_id='1-248367227-TC-T') self.assertEqual(saved_variant.key, 100) self.assertDictEqual(saved_variant.genotypes, new_family_genotypes) - self.assertDictEqual(saved_variant.saved_variant_json, {}) def test_get_hpo_terms(self): url = reverse(get_hpo_terms, args=['HP:0011458']) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index 16ce8a0629..bc13166699 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -57,14 +57,14 @@ 'parsed_file_data': RNA_OUTLIER_SAMPLE_DATA, 'required_columns': RNA_OUTLIER_REQUIRED_COLUMNS, 'rows': [ - 'sampleID\tgeneID\tFDR set\tpValue\tpadjust\tzScore', - 'NA19675_1\tENSG00000240361\tdetail1\t0.01\t0.13\t-3.1', - 'NA19675_1\tENSG00000240361\tdetail2\t0.01\t0.13\t-3.1', - 'NA19675_1\tENSG00000233750\tdetail1\t0.064\t0.0000057\t7.8', - 'NA21234\tENSG00000233750\tdetail1\t0.064\t0.0000057\t7.8', - 'HG00731\tENSG00000240361\t\t0.04\t0.112\t1.9', - 'NA21234\tNOT_A_GENE_ID1\tdetail1\t0.064\t0.0000057\t7.8', - 'NA21234\t\tdetail1\t0.064\t0.0000057\t7.8', + 'sampleID,geneID,FDR set,pValue,padjust,zScore', + 'NA19675_1,ENSG00000240361,detail1,0.01,0.13,-3.1', + 'NA19675_1,ENSG00000240361,detail2,0.01,0.13,-3.1', + 'NA19675_1,ENSG00000233750,detail1,0.064,0.0000057,7.8', + 'NA21234,ENSG00000233750,detail1,0.064,0.0000057,7.8', + 'HG00731,ENSG00000240361,,0.04,0.112,1.9', + 'NA21234,NOT_A_GENE_ID1,detail1,0.064,0.0000057,7.8', + 'NA21234,,detail1,0.064,0.0000057,7.8', ], 'message_data_type': 'Expression Outlier', }, @@ -75,11 +75,11 @@ 'required_columns': RNA_TPM_REQUIRED_COLUMNS, 'mismatch_field': 'tpm', 'rows': [ - 'Name\tDescription\tNA19675_1', - 'ENSG00000240361\tsome gene of interest\t7.8', - 'ENSG00000233750\t\t0.0', - 'NOT_A_GENE_ID1\t\t0.064', - '\t\t0.064', + 'Name,Description,NA19675_1', + 'ENSG00000240361,some gene of interest,7.8', + 'ENSG00000233750,,0.0', + 'NOT_A_GENE_ID1,,0.064', + ',,0.064', ], 'message_data_type': 'Expression', }, @@ -94,13 +94,13 @@ 'row_id': 'ENSG00000233750-2-167254166-167258349-*-psi3', 'invalid_format_field': 'p_value', 'rows': [ - 'hgncSymbol\tseqnames\tstart\tend\tstrand\tsampleID\ttype\tpValue\tpadjust\tdeltaPsi\tcounts\tmeanCounts\ttotalCounts\tmeanTotalCounts\tnonsplitCounts', - 'ENSG00000233750;ENSG00000240361\tchr2\t167254166\t167258349\t*\tNA19675_1\tpsi3\t1.56E-25\t-4.9\t-0.46\t166\t16.6\t1660\t1.66\t1', - 'ENSG00000240361\tchr7\t132885746\t132975168\t*\tNA19675_1\tpsi5\t1.08E-56\t-6.53\t-0.85\t231\t0.231\t2313\t231.3\t1', - 'ENSG00000233750\tchr2\t167258096\t167258349\t*\tNA21234\tpsi3\t1.56E-25\t6.33\t0.45\t143\t14.3\t1433\t143.3\t1', - '\tchr2\t167258096\t167258349\t*\tHG00731\tpsi3\t1.56E-25\t6.33\t0.45\t143\t14.3\t1433\t143.3\t1', - 'NOT_A_GENE_ID1\tchr2\t167258096\t167258349\t*\tNA21234\tpsi3\t1.56E-25\t6.33\t0.45\t143\t14.3\t1433\t143.3\t1', - '\tchr2\t167258096\t167258349\t*\tNA19675_1\tpsi3\t1.56E-25\t6.33\t0.45\t143\t14.3\t1433\t143.3\t1', + 'hgncSymbol,seqnames,start,end,strand,sampleID,type,pValue,padjust,deltaPsi,counts,meanCounts,totalCounts,meanTotalCounts,nonsplitCounts', + 'ENSG00000233750;ENSG00000240361,chr2,167254166,167258349,*,NA19675_1,psi3,1.56E-25,-4.9,-0.46,166,16.6,1660,1.66,1', + 'ENSG00000240361,chr7,132885746,132975168,*,NA19675_1,psi5,1.08E-56,-6.53,-0.85,231,0.231,2313,231.3,1', + 'ENSG00000233750,chr2,167258096,167258349,*,NA21234,psi3,1.56E-25,6.33,0.45,143,14.3,1433,143.3,1', + ',chr2,167258096,167258349,*,HG00731,psi3,1.56E-25,6.33,0.45,143,14.3,1433,143.3,1', + 'NOT_A_GENE_ID1,chr2,167258096,167258349,*,NA21234,psi3,1.56E-25,6.33,0.45,143,14.3,1433,143.3,1', + ',chr2,167258096,167258349,*,NA19675_1,psi3,1.56E-25,6.33,0.45,143,14.3,1433,143.3,1', ], 'message_data_type': 'Splice Outlier', } @@ -729,7 +729,7 @@ def _test_update_project_rna(self, data_type, mock_subprocess, mock_does_file_ex self.login_manager() # Test errors - file = f'{self.TEMP_DIR}/new_samples.tsv.gz' + file = f'{self.TEMP_DIR}/new_samples.csv.gz' body = {'dataType': data_type, 'file': file, 'tissue': tissue, 'sequencingType': sequencing_type} self._set_file_not_found(file, mock_subprocess, mock_does_file_exist, mock_open) self.reset_logs() @@ -739,7 +739,7 @@ def _test_update_project_rna(self, data_type, mock_subprocess, mock_does_file_ex mock_subprocess.return_value.wait.return_value = 0 self._set_local_file_iter([], mock_does_file_exist, mock_open) - invalid_file_ext = file.replace('tsv.gz', 'xlsx') + invalid_file_ext = file.replace('csv.gz', 'xlsx') invalid_body = {**body, 'file': invalid_file_ext} response = self.client.post(url, content_type='application/json', data=json.dumps(invalid_body)) self.assertEqual(response.status_code, 400) @@ -785,7 +785,7 @@ def _test_update_project_rna(self, data_type, mock_subprocess, mock_does_file_ex # test database models are correct self.assertEqual(model_cls.objects.count(), initial_model_count - initial_sample_model_count) rna_samples = RnaSample.objects.filter( - tissue_type=tissue, data_type=data_type, data_source='new_samples.tsv.gz', is_active=False, sequencing_type=sequencing_type, + tissue_type=tissue, data_type=data_type, data_source='new_samples.csv.gz', is_active=False, sequencing_type=sequencing_type, ) self.assertEqual(rna_samples.count(), 1 if single_sample_file else 2) guid_map = {'NA19675_1': rna_samples.get(individual_id=1).guid} @@ -795,7 +795,7 @@ def _test_update_project_rna(self, data_type, mock_subprocess, mock_does_file_ex # test notifications subprocess_logs = self._get_expected_read_file_subprocess_calls([ - f'gsutil cp gs://seqr-scratch-temp/new_samples.tsv.gz tmp/temp_uploads/{file_path}', + f'gsutil cp gs://seqr-scratch-temp/new_samples.csv.gz tmp/temp_uploads/{file_path}', f'gsutil mv tmp/temp_uploads/{file_path}/*.json.gz gs://seqr-scratch-temp/{file_path}', ], mock_subprocess, wait_command=True) self.assert_json_logs(self.manager_user, subprocess_logs[:1] + [ @@ -1068,6 +1068,7 @@ def _check_created_project_groups(self, project): self.assertSetEqual(set(project.can_edit_group.user_set.all()), {self.pm_user}) self.assertSetEqual(set(project.can_view_group.user_set.all()), {self.pm_user}) + @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') def test_update_project_workspace(self): url = reverse(update_project_workspace, args=[PROJECT_GUID]) # For non-AnVIL seqr, updating workspace should always fail diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 60d214a668..2edbc90742 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -4,7 +4,7 @@ import responses from settings import AIRTABLE_URL -from seqr.models import Project, SavedVariant, RnaSample +from seqr.models import Project, RnaSample from seqr.views.apis.report_api import seqr_stats, anvil_export, gregor_export, family_metadata, variant_metadata from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest @@ -824,7 +824,7 @@ def _test_gregor_export(self, url, mock_subprocess, mock_temp_dir, mock_open, mo match=[responses.matchers.query_param_matcher({'fields[]': 'SMID'}, strict_match=False)] ) responses.add( - responses.GET, '{}/app3Y97xtbbaOopVR/GREGoR Data Model'.format(AIRTABLE_URL), json=AIRTABLE_GREGOR_RECORDS, + responses.GET, '{}/app3Y97xtbbaOopVR/GREGoR Data Model'.format(AIRTABLE_URL), json={'records': AIRTABLE_GREGOR_RECORDS['records'][:2]+AIRTABLE_GREGOR_RECORDS['records'][3:]}, status=200) responses.add(responses.GET, MOCK_DATA_MODEL_URL, status=404) @@ -856,7 +856,7 @@ def _test_gregor_export(self, url, mock_subprocess, mock_temp_dir, mock_open, mo recommended_warnings = [ 'The following entries are missing RNA airtable data: NA19675', - 'The following entries are missing WES airtable data: NA19675, NA19679', + 'The following entries are missing WES airtable data: NA19675', 'The following entries have WGS airtable data but do not have equivalent loaded data in seqr, so airtable data is omitted: NA19675, NA20888, VCGS_FAM203_621', 'The following entries are missing recommended "recontactable" in the "participant" table: Broad_HG00731, Broad_HG00732, Broad_HG00733, Broad_NA19678, Broad_NA20870, Broad_NA20872, Broad_NA20874, Broad_NA20875, Broad_NA20876, Broad_NA20881', 'The following entries are missing recommended "reported_race" in the "participant" table: Broad_HG00733, Broad_NA19678, Broad_NA19679, Broad_NA20870, Broad_NA20872, Broad_NA20874, Broad_NA20875, Broad_NA20876, Broad_NA20881, Broad_NA20888', @@ -875,6 +875,7 @@ def _test_gregor_export(self, url, mock_subprocess, mock_temp_dir, mock_open, mo f'No data model found for "{file}" table' for file in reversed(EXPECTED_GREGOR_FILES) if file not in INVALID_MODEL_TABLES ] + [ missing_participant_error, + 'The following entries are missing airtable metadata: NA19679', 'The following tables are required in the data model but absent from the reports: subject, dna_read_data_set', ] + [ 'The following entries are missing required "prior_testing" in the "participant" table: Broad_HG00731, Broad_HG00732', @@ -885,17 +886,23 @@ def _test_gregor_export(self, url, mock_subprocess, mock_temp_dir, mock_open, mo 'The following entries are missing required "mean_coverage" (from Airtable) in the "aligned_dna_short_read" table: Broad_exome_VCGS_FAM203_621_D2_1', 'The following entries have non-unique values for "alignment_software" (from Airtable) in the "aligned_dna_short_read" table: BWA-MEM-2.3 (Broad_exome_NA20888_1, Broad_exome_VCGS_FAM203_621_D2_1)', 'The following entries have invalid values for "analysis_details" (from Airtable) in the "aligned_dna_short_read" table. Allowed values are a google bucket path starting with gs://. Invalid values: Broad_exome_VCGS_FAM203_621_D2_1 (DOI:10.5281/zenodo.4469317)', - 'The following entries have invalid values for "date_data_generation" (from Airtable) in the "experiment_rna_short_read" table. Allowed values have data type float. Invalid values: NA19679 (2023-02-11)', 'The following entries are missing required "experiment_id" (from Airtable) in the "genetic_findings" table: Broad_NA19675_1_21_3343353', 'The following entries have non-unique values for "experiment_id" (from Airtable) in the "genetic_findings" table: Broad_exome_VCGS_FAM203_621_D2 (Broad_HG00731_19_1912632, Broad_HG00731_1_248367227)', ] self.assertListEqual(response.json()['errors'], validation_errors) + responses.replace( + responses.GET, '{}/app3Y97xtbbaOopVR/GREGoR Data Model'.format(AIRTABLE_URL), json=AIRTABLE_GREGOR_RECORDS, + ) mock_open.reset_mock() response = self.client.post( url, content_type='application/json', data=json.dumps({**body, 'overrideValidation': True}) ) self.assertEqual(response.status_code, 200) + validation_errors.pop(8) + validation_errors.insert(17, 'The following entries have invalid values for "date_data_generation" (from Airtable) in the "experiment_rna_short_read" table. Allowed values have data type float. Invalid values: NA19679 (2023-02-11)') + recommended_warnings[1] += ', NA19679' + validation_warnings[4] = recommended_warnings[1] expected_response = { 'info': ['Successfully validated and uploaded Gregor Report for 9 families'], 'warnings': validation_errors + validation_warnings, @@ -963,11 +970,6 @@ def _test_gregor_export(self, url, mock_subprocess, mock_temp_dir, mock_open, mo project.consent_code = 'H' project.save() - # For SV variant, test reports in gene associated with OMIM condition even if not annotated - variant = SavedVariant.objects.get(id=7) - variant.saved_variant_json['transcripts'] = {'ENSG00000135953': []} - variant.save() - responses.calls.reset() responses.add(responses.GET, 'https://monarchinitiative.org/v3/api/entity/MONDO:0008788', status=200, json={ 'id': 'MONDO:0008788', diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 81a3c0c713..099d210914 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -56,34 +56,28 @@ def create_manual_saved_variant_handler(request, family_guid): tags = variant_json.pop('tags', []) saved_variant_guids = {guid for guid, is_selected in variant_json.pop('variants', {}).items() if is_selected} - genome_version = family.project.genome_version try: xpos = get_xpos(variant_json['chrom'], variant_json['pos']) variant_id = variant_json.get('svName') or f"{variant_json['chrom']}-{variant_json['pos']}-{variant_json['ref']}-{variant_json['alt']}" except (KeyError, ValueError) as e: return create_json_response({'error': str(e)}, status=400) - variant_json.update({ - 'genomeVersion': genome_version, - 'transcripts': {}, - 'variantId': variant_id, - 'xpos': xpos, - }) - gene_id = variant_json.pop('geneId', None) - if gene_id: - variant_json['transcripts'][gene_id] = [] - if variant_json.get('mainTranscriptId'): - variant_json['transcripts'][gene_id].append({ - 'transcriptId': variant_json['mainTranscriptId'], - 'hgvsc': variant_json.pop('hgvsc', None), - 'hgvsp': variant_json.pop('hgvsp', None), - }) + main_transcript = {} + if variant_json.get('mainTranscriptId'): + main_transcript = { + 'transcriptId': variant_json['mainTranscriptId'], + 'hgvsc': variant_json.pop('hgvsc', None), + 'hgvsp': variant_json.pop('hgvsp', None), + } - variant_json['saved_variant_json'] = {**variant_json} variant_json.update({ + 'variantId': variant_id, + 'xpos': xpos, 'key': None, 'dataset_type': Dataset.DATASET_TYPE_SV_CALLS if variant_json.get('svName') else Dataset.DATASET_TYPE_VARIANT_CALLS, + 'gene_ids': [gene_id] if gene_id else [], + 'main_transcript': main_transcript, }) model_json = parse_saved_variant_json(variant_json, family.id) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 6ed916b117..954ecb7d03 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -177,11 +177,34 @@ def test_saved_variant_data(self): variants = response_json['variantsById'] self.assertSetEqual(set(variants.keys()), {'1-248367227-TC-T', '21-3343353-GAGA-G'}) variant = variants['21-3343353-GAGA-G'] - variant_fields = {*SAVED_VARIANT_DETAIL_FIELDS, 'mainTranscriptId'} + variant_fields = { + *SAVED_VARIANT_FIELDS, 'mainTranscriptId', 'genomeVersion', 'tagGuids', 'functionalDataGuids', 'noteGuids', + 'genotypes', 'transcripts', 'acmgClassification', + } self.assertSetEqual(set(variant.keys()), variant_fields) self.assertListEqual(variant['familyGuids'], ['F000001_1']) self.assertSetEqual(set(variant['genotypes'].keys()), {'I000003_na19679', 'I000001_na19675', 'I000002_na19678'}) + discovery_tags = [{ + 'savedVariant': { + 'variantGuid': 'SV0000006_1248367227_r0003_tes', + 'familyGuid': 'F000012_12', + 'projectGuid': 'R0003_test', + }, + 'tagGuid': 'VT1726961_2103343353_r0003_tes', + 'name': 'Tier 1 - Novel gene and phenotype', + 'category': 'CMG Discovery Tags', + 'color': '#03441E', + 'searchHash': None, + 'searchName': None, + 'metadata': None, + 'lastModifiedDate': '2018-05-29T16:32:51.449Z', + 'createdBy': None, + }] + self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], discovery_tags) + self.assertEqual(variants['1-248367227-TC-T']['noAccessDiscoveryFamilies'], 1) + self.assertListEqual(variants['1-248367227-TC-T']['familyGuids'], ['F000002_2']) + tag = response_json['variantTagsByGuid']['VT1708633_2103343353_r0390_100'] self.assertSetEqual(set(tag.keys()), TAG_FIELDS) self.assertDictEqual(tag, { @@ -225,7 +248,9 @@ def test_saved_variant_data(self): 'spliceOutliers': {}, }}) - self.assertDictEqual(response_json['familiesByGuid'], {'F000001_1': {'tpmGenes': ['ENSG00000135953']}}) + self.assertSetEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1', 'F000012_12'}) + self.assertSetEqual(set(response_json['familiesByGuid']['F000012_12'].keys()), FAMILY_FIELDS) + self.assertDictEqual(response_json['familiesByGuid']['F000001_1'], {'tpmGenes': ['ENSG00000135953']}) self.assertDictEqual(response_json['omimIntervals'], {}) @@ -263,7 +288,7 @@ def test_saved_variant_data(self): family_context_response_keys.update(SAVED_VARIANT_RESPONSE_KEYS) self.assertSetEqual(set(response_json.keys()), family_context_response_keys) self.assertEqual(len(response_json['savedVariantsByGuid']), 2) - self.assertEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1', 'F000002_2'}) + self.assertEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1', 'F000002_2', 'F000012_12'}) family_fields = {'individualGuids', 'tpmGenes'} family_fields.update(FAMILY_FIELDS) self.assertSetEqual(set(response_json['familiesByGuid']['F000001_1'].keys()), family_fields) @@ -314,7 +339,7 @@ def test_saved_variant_data(self): response = self.client.get(url.replace(PROJECT_GUID, 'R0003_test')) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), no_families_response_keys) + self.assertSetEqual(set(response_json.keys()), {*no_families_response_keys, 'familiesByGuid'}) self.assertSetEqual( set(response_json['savedVariantsByGuid'].keys()), @@ -330,52 +355,8 @@ def test_saved_variant_data(self): }}) self.assertDictEqual(response_json['rnaSeqData'], {}) - # Test cross-project discovery for analyst users - self.login_analyst_user() - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS) - self.assertSetEqual( - set(response_json['savedVariantsByGuid'].keys()), - {'SV0000002_1248367227_r0390_100', VARIANT_GUID} - ) - variants = response_json['variantsById'] - self.assertSetEqual(set(variants.keys()), {'1-248367227-TC-T', '21-3343353-GAGA-G'}) - discovery_tags = [{ - 'savedVariant': { - 'variantGuid': 'SV0000006_1248367227_r0003_tes', - 'familyGuid': 'F000012_12', - 'projectGuid': 'R0003_test', - }, - 'tagGuid': 'VT1726961_2103343353_r0003_tes', - 'name': 'Tier 1 - Novel gene and phenotype', - 'category': 'CMG Discovery Tags', - 'color': '#03441E', - 'searchHash': None, - 'searchName': None, - 'metadata': None, - 'lastModifiedDate': '2018-05-29T16:32:51.449Z', - 'createdBy': None, - }] - self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], discovery_tags) - self.assertListEqual(variants['1-248367227-TC-T']['familyGuids'], ['F000002_2']) - self.assertSetEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1', 'F000012_12'}) - self.assertSetEqual(set(response_json['familiesByGuid']['F000012_12'].keys()), FAMILY_FIELDS) - self.assertDictEqual(response_json['familiesByGuid']['F000001_1'], {'tpmGenes': ['ENSG00000135953']}) - - # Test discovery tags with family context - response = self.client.get(load_family_context_url) - self.assertEqual(response.status_code, 200) - response_json = response.json() - self.assertSetEqual(set(response_json.keys()), family_context_response_keys) - variants = response_json['variantsById'] - self.assertSetEqual(set(variants.keys()), {'1-248367227-TC-T', '21-3343353-GAGA-G'}) - self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], discovery_tags) - self.assertListEqual(variants['1-248367227-TC-T']['familyGuids'], ['F000002_2']) - self.assertEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1', 'F000002_2', 'F000012_12'}) - # Test empty project + self.login_analyst_user() empty_project_url = url.replace(PROJECT_GUID, 'R0002_empty') response = self.client.get(empty_project_url) self.assertEqual(response.status_code, 200) @@ -391,10 +372,10 @@ def test_saved_variant_data(self): response = self.client.get(url) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS - {'omimIntervals'}) + self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS - {'omimIntervals', 'transcriptsById'}) self.assertSetEqual(set(response_json['savedVariantsByGuid']['SV0000002_1248367227_r0390_100'].keys()), fields) self.assertSetEqual(set(response_json['variantsById']['1-248367227-TC-T'].keys()), { - *variant_fields, 'discoveryTags', 'screenRegionType', 'sortedRegulatoryFeatureConsequences', 'sortedMotifFeatureConsequences', + *variant_fields, *SAVED_VARIANT_DETAIL_FIELDS, 'discoveryTags', 'noAccessDiscoveryFamilies', 'screenRegionType', 'sortedRegulatoryFeatureConsequences', 'sortedMotifFeatureConsequences', }) def test_create_saved_variant(self): @@ -449,7 +430,7 @@ def test_create_saved_variant(self): self.assertEqual(response.status_code, 200) self.assertListEqual(list(response.json()['savedVariantsByGuid'].keys()), [variant_guid]) - def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, dataset_type='SNV_INDEL', sv_type=None, main_transcript=None, has_saved_variant_json=False): + def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, dataset_type='SNV_INDEL', sv_type=None, main_transcript=None): for field in ['xpos', 'ref', 'alt', 'key']: self.assertEqual(variant_json.get(field), getattr(saved_variant, field, None)) self.assertEqual(saved_variant.gene_ids, gene_ids or []) @@ -457,7 +438,6 @@ def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, da self.assertEqual(dataset_type, saved_variant.dataset_type) self.assertEqual(sv_type, saved_variant.sv_type) self.assertDictEqual(main_transcript or {}, saved_variant.main_transcript) - self.assertDictEqual(variant_json if has_saved_variant_json else {}, saved_variant.saved_variant_json) def test_create_saved_sv_variant(self): # SVs are only supported on build 38 @@ -628,10 +608,9 @@ def test_create_manual_variant(self): 'transcripts': {'ENSG00000277258': [{'hgvsc': 'c.156GAG>A', 'hgvsp': 'p.Leu52Phe', 'transcriptId': 'ENST00000459627'}]}, **{k: v for k, v in manual_variant_request_body.items() if k in {'alt', 'ref', 'chrom', 'pos', 'genotypes', 'mainTranscriptId'}}, } - self._assert_created_variant(saved_variant, variant_json, gene_ids=['ENSG00000277258'], has_saved_variant_json=True, main_transcript={ + self._assert_created_variant(saved_variant, variant_json, gene_ids=['ENSG00000277258'], main_transcript={ 'hgvsc': 'c.156GAG>A', 'hgvsp': 'p.Leu52Phe', 'transcriptId': 'ENST00000459627', }) - self.assertDictEqual(variant_json, saved_variant.saved_variant_json) base_variant_json = { 'variantGuid': variant_guid, @@ -684,7 +663,7 @@ def test_create_manual_variant(self): 'transcripts': {'ENSG00000240361': []}, **{k: v for k, v in manual_variant_request_body.items() if k in {'chrom', 'pos', 'end', 'svName', 'svType', 'genotypes'}}, } - self._assert_created_variant(saved_sv_variant, sv_variant_json, gene_ids=['ENSG00000240361'], dataset_type='SV', sv_type='DEL', has_saved_variant_json=True) + self._assert_created_variant(saved_sv_variant, sv_variant_json, gene_ids=['ENSG00000240361'], dataset_type='SV', sv_type='DEL') sv_variant_json = { **{k: v for k, v in sv_variant_json.items() if k in SAVED_VARIANT_FIELDS}, @@ -1153,7 +1132,7 @@ def test_update_variant_acmg_classification(self): # Tests for AnVIL access disabled class LocalSavedVariantAPITest(AuthenticationTestCase, SavedVariantAPITest): - fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_saved_variants'] + fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_discovery_variants', 'clickhouse_saved_variants'] def assert_no_list_ws_has_al(self, acl_call_count): @@ -1166,7 +1145,7 @@ def assert_no_list_ws_has_al(self, acl_call_count): # Test for permissions from AnVIL only class AnvilSavedVariantAPITest(AnvilAuthenticationTestCase, SavedVariantAPITest): - fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'clickhouse_saved_variants'] + fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'clickhouse_discovery_variants', 'clickhouse_saved_variants'] @classmethod def setUpTestData(cls): @@ -1181,9 +1160,9 @@ def test_saved_variant_data(self, *args): mock.ANY, 'ext-data', 'empty') self.mock_get_ws_access_level.assert_called_with( mock.ANY, 'my-seqr-billing', 'anvil-1kg project n\u00e5me with uni\u00e7\u00f8de') - self.assertEqual(self.mock_get_ws_access_level.call_count, 18) - self.mock_get_groups.assert_has_calls([mock.call(self.collaborator_user), mock.call(self.analyst_user)]) - self.assertEqual(self.mock_get_groups.call_count, 12) + self.assertEqual(self.mock_get_ws_access_level.call_count, 15) + self.mock_get_groups.assert_called_with(self.collaborator_user) + self.assertEqual(self.mock_get_groups.call_count, 1) self.mock_get_ws_acl.assert_not_called() self.mock_get_group_members.assert_not_called() diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index 8677bda356..5f15f51aa0 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -132,7 +132,7 @@ 'projectGuid': 'R0004_non_analyst_project', 'internal_project_id': 'Non-Analyst Project', 'affected_status': 'Affected', - 'analysisStatus': 'Rncc', + 'analysisStatus': 'S_ng', 'ancestry': '', 'consanguinity': 'Unknown', 'data_type': 'WGS', @@ -154,7 +154,7 @@ 'proband_relationship': 'Self', 'sex': 'Female', 'sex_detail': None, - 'solve_status': 'Unsolved', + 'solve_status': 'Solved', 'alt-1': 'T', 'chrom-1': '1', 'gene_known_for_phenotype-1': 'Candidate', @@ -616,6 +616,11 @@ def test_bulk_update_family_external_analysis(self, mock_load_uploaded_file, moc {'4': {'name': 'De-Novo', 'date': '2023-12-05'}, 'support': {'name': 'High in Silico Scores', 'date': '2023-12-05'}}, ) + # Test reloading skips unchanged tags + response = self.client.post(url, content_type='application/json', data=json.dumps(body)) + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), {'info': ['Loaded 0 new and 1 updated AIP tags for 2 families (skipped 2 unchanged tags)']}) + self.check_no_analyst_no_access(url) def _assert_expected_new_saved_variant(self, new_saved_variant): @@ -836,7 +841,7 @@ def test_mme_details(self, *args): def test_saved_variants_page(self): super(AnvilSummaryDataAPITest, self).test_saved_variants_page() assert_has_expected_calls(self, [ - self.no_access_user, self.manager_user, self.manager_user, self.manager_user, self.analyst_user, self.analyst_user + self.no_access_user, self.manager_user, self.manager_user, self.manager_user, self.manager_user, self.manager_user, self.analyst_user, self.analyst_user ], skip_group_call_idxs=[2]) self.mock_get_ws_access_level.assert_called_with( self.analyst_user, 'my-seqr-billing', 'anvil-1kg project nåme with uniçøde') diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 85875748b6..ded007b9a8 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -16,8 +16,8 @@ from clickhouse_search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ get_sorted_search_results, clickhouse_variant_lookup, InvalidSearchException -from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_LOOKUP -from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Dataset +from reference_data.models import HumanPhenotypeOntology, GENOME_VERSION_GRCh38, GENOME_VERSION_LOOKUP +from seqr.models import Project, Family, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Dataset from seqr.views.utils.export_utils import export_table from seqr.utils.gene_utils import get_genes_for_variant_display from seqr.utils.logging_utils import SeqrLogger @@ -27,7 +27,7 @@ from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json from seqr.views.utils.orm_to_json_utils import get_json_for_saved_variants_with_tags, get_json_for_saved_search,\ - get_json_for_saved_searches, add_individual_hpo_details, FAMILY_ADDITIONAL_VALUES + get_json_for_saved_searches, FAMILY_ADDITIONAL_VALUES from seqr.views.utils.permissions_utils import check_project_permissions, get_project_guids_user_can_view, \ login_and_policies_required, check_user_created_object_permissions, check_projects_view_permission, user_is_analyst from seqr.views.utils.project_context_utils import get_projects_child_entities @@ -37,17 +37,6 @@ logger = SeqrLogger(__name__) -GENOTYPE_AC_LOOKUP = { - 'ref_ref': [0, 0], - 'has_ref': [0, 1], - 'ref_alt': [1, 1], - 'has_alt': [1, 2], - 'alt_alt': [2, 2], -} -AFFECTED = Individual.AFFECTED_STATUS_AFFECTED -UNAFFECTED = Individual.AFFECTED_STATUS_UNAFFECTED - - @login_and_policies_required def query_variants_handler(request, search_hash): page = int(request.GET.get('page') or 1) @@ -352,8 +341,8 @@ def export_variants_handler(request, search_hash): variants = split_variants - max_families_per_variant = max([len(variant.get('familyGuids', [1])) for variant in variants]) - max_samples_per_variant = max([len(variant.get('genotypes', {})) for variant in variants]) + max_families_per_variant = max([len(variant.get('familyGuids', [1])) for variant in variants] or [0]) + max_samples_per_variant = max([len(variant.get('genotypes', {})) for variant in variants] or [0]) rows = [] for variant in variants: @@ -598,15 +587,19 @@ def variant_lookup_handler(request): family_guids = set() for variant in variants: family_guids.update(variant['familyGenotypes'].keys()) + family_guids.update(variant['discoveryFamilies']) + family_guids.update(variant['excludedTagFamilies']) - families = Family.objects.filter( + family_guids = set(Family.objects.filter( guid__in=family_guids, project__guid__in=get_project_guids_user_can_view(request.user, limit_data_manager=True), - ) + ).values_list('guid', flat=True)) for variant in variants: - variant['familyGuids'] = list(families.values_list('guid', flat=True)) + variant['familyGuids'] = family_guids.intersection(variant['familyGenotypes'].keys()) + variant['discoveryTagFamilies'] = set(variant['discoveryFamilies']) - family_guids + variant['excludedTagFamilies'] = set(variant['excludedTagFamilies']) - family_guids - saved_variants = _get_saved_variant_models(variants) if families else None + saved_variants = _get_saved_variant_models(variants) if family_guids else None response = get_variants_response( request, saved_variants=saved_variants, response_variants=variants, add_all_context=True, add_locus_list_detail=True, genome_version=genome_version, @@ -631,22 +624,13 @@ def _get_lookup_cache_key(user, variant_id, sample_type, genome_version, affecte def _update_lookup_variant(variant, response, individual_guid_map, user): - no_access_families = set(variant['familyGenotypes']) - set(variant['familyGuids']) - individual_summary_map = { - (i.pop('family__guid'), i.pop('individual_id')): (i.pop('guid'), i) - for i in Individual.objects.filter(family__guid__in=no_access_families).values( - 'family__guid', 'individual_id', 'affected', 'sex', 'features', 'guid', - vlmContactEmail=F('family__project__vlm_contact_email'), - restrict_sharing=F('family__project__restrict_sharing'), - ) - } - add_individual_hpo_details([i for _, i in individual_summary_map.values()]) - variant['genotypes'] = {} variant['lookupFamilyGuids'] = sorted([guid for guid in variant.pop('familyGuids') if guid in variant['familyGenotypes']]) variant['familyGuids'] = [] + variant.pop('noAccessDiscoveryFamilies', None) for family_guid in variant['lookupFamilyGuids']: for genotype in variant['familyGenotypes'].pop(family_guid): + genotype.pop('metadata', None) individual_guid = individual_guid_map.get((family_guid, genotype['sampleId'])) if not individual_guid: logger.error( @@ -659,36 +643,32 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): genotype = [variant['genotypes'][individual_guid], genotype] variant['genotypes'][individual_guid] = genotype + all_feature_ids = set() + family_guid_map = {} for i, (unmapped_family_guid, genotypes) in enumerate(sorted(variant.pop('familyGenotypes').items())): family_guid = f'F{i}_{variant["variantId"]}' + family_guid_map[unmapped_family_guid] = family_guid variant['lookupFamilyGuids'].append(family_guid) if unmapped_family_guid in variant.get('liftedFamilyGuids', []): variant['liftedFamilyGuids'][variant['liftedFamilyGuids'].index(unmapped_family_guid)] = family_guid - individual_guid_map = {} + individual_key_map = {} for j, genotype in enumerate(genotypes): individual_key = (genotype.pop('familyGuid'), genotype.pop('sampleId')) - if individual_key not in individual_summary_map: - logger.error( - f'Unable to map sample {individual_key[1]} in family {individual_key[0]} to an individual for variant {variant["variantId"]}', - user, - ) - continue - unmapped_individual_guid, individual = individual_summary_map[individual_key] - if unmapped_individual_guid in individual_guid_map: - individual_guid = individual_guid_map[unmapped_individual_guid] + individual = genotype.pop('metadata', {}) + if individual_key in individual_key_map: + individual_guid = individual_key_map[individual_key] variant['genotypes'][individual_guid] = [variant['genotypes'][individual_guid], genotype] continue individual_guid = f'I{j}_{family_guid}' - individual_guid_map[unmapped_individual_guid] = individual_guid - features = individual['features'] or [] - if individual.pop('restrict_sharing'): - feature_category_count = defaultdict(int) - for feature in features: - feature_category_count[feature.get('category', 'Other')] += 1 - features = [ - {'category': category, 'label': f'{count} terms'} - for category, count in feature_category_count.items() - ] + individual_key_map[individual_key] = individual_guid + features = json.loads(individual['features']) if individual.get('features') else [] + all_feature_ids.update([feature['id'] for feature in features]) + omim_id = individual.pop('omim_id') + mondo_id = individual.pop('mondo_id') + if individual.get('restrict_sharing'): + individual.pop('isSolved') + else: + individual['disease'] = f'OMIM:{omim_id}' if omim_id else mondo_id response['individualsByGuid'][individual_guid] = { **individual, 'familyGuid': family_guid, @@ -697,6 +677,34 @@ def _update_lookup_variant(variant, response, individual_guid_map, user): } variant['genotypes'][individual_guid] = genotype + variant['discoveryTagFamilies'] = sorted([ + family_guid_map[family_guid] for family_guid in variant['discoveryTagFamilies'] if family_guid in family_guid_map + ]) + variant['excludedTagFamilies'] = sorted([ + family_guid_map[family_guid] for family_guid in variant['excludedTagFamilies'] if family_guid in family_guid_map + ]) + + _parse_hpo_terms(all_feature_ids, response['individualsByGuid'].values()) + + +def _parse_hpo_terms(all_feature_ids, individuals): + hpo_terms_by_id = { + hpo.pop('hpo_id'): hpo for hpo in HumanPhenotypeOntology.objects.filter(hpo_id__in=all_feature_ids).values( + 'hpo_id', category=F('category_id'), label=F('name'), + ) + } + for individual in individuals: + for feature in individual['features'] or []: + feature.update(hpo_terms_by_id.get(feature['id'], {})) + if individual.pop('restrict_sharing', False) and individual['features']: + feature_category_count = defaultdict(int) + for feature in individual['features']: + feature_category_count[feature.get('category', 'Other')] += 1 + individual['features'] = [ + {'category': category, 'label': f'{count} terms'} + for category, count in feature_category_count.items() + ] + @login_and_policies_required def vlm_lookup_handler(request): diff --git a/seqr/views/utils/airtable_utils.py b/seqr/views/utils/airtable_utils.py index 3d57e5d5f3..4baebea0a2 100644 --- a/seqr/views/utils/airtable_utils.py +++ b/seqr/views/utils/airtable_utils.py @@ -75,8 +75,10 @@ def safe_patch_records(self, record_type, record_or_filters, record_and_filters, and_filters=record_and_filters, page_size=max_records + 1, ) - if not records or len(records) > max_records: - error = f'''Unable to identify Airtable "{record_type}" record to update + except Exception: + records = [] + if not records or len(records) > max_records: + error = f'''Unable to identify Airtable "{record_type}" record to update Record lookup criteria: ``` @@ -88,12 +90,10 @@ def safe_patch_records(self, record_type, record_or_filters, record_and_filters, ``` {json.dumps(update)} ```''' - safe_post_to_slack(SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, error) - return + safe_post_to_slack(SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, error) + return - self.safe_patch_records_by_id(record_type, list(records.keys()), update, error_detail=error_detail) - except Exception as e: - logger.error(f'Airtable patch "{record_type}" error: {e}', self._user, detail=error_detail) + self.safe_patch_records_by_id(record_type, list(records.keys()), update, error_detail=error_detail) def safe_patch_records_by_id(self, record_type, record_ids, update, error_detail=None): self._safe_bulk_update_records( diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index 6007ff8d6d..ea21b57f45 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -126,17 +126,11 @@ def _update_from_record(record, user, families_by_id, individual_lookup, updated record['displayName'] = '' # Update the parent ids last, so if they are referencing updated individuals they will check for the correct ID - if 'father' in record or 'mother' in record: + if record.get(JsonConstants.MATERNAL_ID_COLUMN) is not None or record.get(JsonConstants.PATERNAL_ID_COLUMN) is not None: parent_updates.append({ 'individual': individual, - 'mother': record.pop('mother', None), - 'father': record.pop('father', None), - }) - elif record.get('maternalId') is not None or record.get('paternalId') is not None: - parent_updates.append({ - 'individual': individual, - 'maternalId': record.pop('maternalId', None), - 'paternalId': record.pop('paternalId', None), + JsonConstants.MATERNAL_ID_COLUMN: record.pop(JsonConstants.MATERNAL_ID_COLUMN, None), + JsonConstants.PATERNAL_ID_COLUMN: record.pop(JsonConstants.PATERNAL_ID_COLUMN, None), }) family_notes = record.pop(JsonConstants.FAMILY_NOTES_COLUMN, None) diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 34d3c2616a..4e15831650 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -8,13 +8,13 @@ from panelapp.models import PaLocusList from reference_data.models import HumanPhenotypeOntology -from seqr.models import GeneNote, VariantNote, VariantTag, VariantFunctionalData, SavedVariant, Family, CAN_VIEW, CAN_EDIT, \ +from seqr.models import GeneNote, VariantNote, VariantTag, VariantFunctionalData, CAN_VIEW, CAN_EDIT, \ get_audit_field_names, RnaSeqOutlier, RnaSeqSpliceOutlier, VariantSearchResults from seqr.utils.xpos_utils import get_chrom_pos from seqr.views.utils.json_utils import _to_camel_case from seqr.views.utils.permissions_utils import has_project_permissions, \ project_has_anvil, get_workspace_collaborator_perms, user_is_analyst, user_is_data_manager, user_is_pm, \ - is_internal_anvil_project, get_project_guids_user_can_view, get_anvil_analyst_user_emails + is_internal_anvil_project, get_anvil_analyst_user_emails from seqr.views.utils.terra_api_utils import is_anvil_authenticated, anvil_enabled from settings import ANALYST_USER_GROUP, SERVICE_ACCOUNT_FOR_ANVIL, MEDIA_URL @@ -366,14 +366,14 @@ def _format_variant_tags(tags): return tags -def get_json_for_saved_variants_child_entities(tag_cls, saved_variant_id_map, tag_filter=None): +def get_json_for_saved_variants_child_entities(tag_cls, saved_variant_id_map, tag_filter=None, ): variant_tag_id_map = defaultdict(list) for savedvariant_id, tag_id in tag_cls.saved_variants.through.objects.filter( savedvariant_id__in=saved_variant_id_map.keys()).values_list( 'savedvariant_id', f'{tag_cls.__name__.lower()}_id', ): variant_tag_id_map[tag_id].append(savedvariant_id) - tag_models = tag_cls.objects.filter(id__in=variant_tag_id_map.keys()) + tag_models = tag_cls.objects.filter(id__in=variant_tag_id_map.keys()).order_by('id') if tag_filter: tag_models = tag_models.filter(**tag_filter) @@ -441,49 +441,6 @@ def get_json_for_saved_variants_with_tags(saved_variants, additional_model_field return response -def get_json_for_discovery_tags(variants, user): - from seqr.views.utils.variant_utils import get_variant_key - response = {} - discovery_tags = defaultdict(list) - - saved_variants = SavedVariant.objects.filter( - variant_id__in={variant['variantId'] for variant in variants}, - family__project__guid__in=get_project_guids_user_can_view(user), - ).only('id', 'guid', 'ref', 'alt', 'xpos', 'family_id').prefetch_related('family', 'family__project') - saved_variants_by_guid = {sv.guid: sv for sv in saved_variants} - saved_variant_id_map = {sv.id: guid for guid, sv in saved_variants_by_guid.items()} - - discovery_tag_json, _ = get_json_for_saved_variants_child_entities( - VariantTag, saved_variant_id_map, tag_filter={'variant_tag_type__category': 'CMG Discovery Tags'}) - if discovery_tag_json: - existing_families = set() - for variant in variants: - existing_families.update(variant['familyGuids']) - - family_ids = set() - for tag in discovery_tag_json: - for variant_guid in tag.pop('variantGuids'): - variant = saved_variants_by_guid[variant_guid] - if variant.family.guid not in existing_families: - family_ids.add(variant.family_id) - tag_json = {'savedVariant': { - 'variantGuid': variant.guid, - 'familyGuid': variant.family.guid, - 'projectGuid': variant.family.project.guid, - }} - tag_json.update(tag) - variant_key = get_variant_key( - genomeVersion=variant.family.project.genome_version, - xpos=variant.xpos, ref=variant.ref, alt=variant.alt, - ) - discovery_tags[variant_key].append(tag_json) - - response['familiesByGuid'] = { - f['familyGuid']: f for f in _get_json_for_families(Family.objects.filter(id__in=family_ids)) - } - return discovery_tags, response - - def get_json_for_variant_note(note): return _get_json_for_model(note, guid_key='noteGuid') diff --git a/seqr/views/utils/pedigree_info_utils.py b/seqr/views/utils/pedigree_info_utils.py index 6fd9c94f36..5e1030a3a6 100644 --- a/seqr/views/utils/pedigree_info_utils.py +++ b/seqr/views/utils/pedigree_info_utils.py @@ -173,9 +173,7 @@ def _parse_header_columns(header, allow_id_update, update_features): column = None full_key = key key = key.lower() - if full_key in JsonConstants.JSON_COLUMNS: - column = full_key - elif key == JsonConstants.FAMILY_NOTES_COLUMN.lower(): + if key == JsonConstants.FAMILY_NOTES_COLUMN.lower(): column = JsonConstants.FAMILY_NOTES_COLUMN elif key.startswith("notes"): column = JsonConstants.NOTES_COLUMN @@ -474,8 +472,6 @@ def _set_proband_relationship(json_records): ) if affected_children: affected = affected_children - if not affected: - continue affected = affected[0] relationships = { @@ -770,30 +766,24 @@ def _get_rgp_dsm_family_notes(row): def _get_rgp_dsm_proband_fields(row): DC = DSMConstants - try: + if row[DC.AGE_COLUMN]: age = int(row[DC.AGE_COLUMN]) birth_year = date.today().year - age - except ValueError: - birth_year = None death_year = None if row[DC.DECEASED_COLUMN] == DC.YES: - try: - age = int(row[DC.DECEASED_AGE_COLUMN]) - death_year = birth_year + age - except (ValueError, TypeError): - death_year = 0 + age = int(row[DC.DECEASED_AGE_COLUMN]) + death_year = birth_year + age - try: + onset_age = None + if row[DC.AGE_OF_ONSET_COLUMN]: onset_age_val = int(row[DC.AGE_OF_ONSET_COLUMN]) onset_age = next(age for cutoff, age in [ (2, 'I'), # Infantile onset (13, 'C'), # Childhood onset (20, 'J'), # Juvenile onset - (200, 'A')# Adult onset + (200, 'A') # Adult onset ] if onset_age_val < cutoff) - except (ValueError, TypeError): - onset_age = None affected_relatives = any( row['{}_{}'.format(parent, DC.AFFECTED_KEY)] == DC.YES for parent in [DC.MOTHER, DC.FATHER] diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index 4319549b77..c03ceeb79a 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -92,11 +92,6 @@ def add_families_context(response, family_models, project_guid, user, is_analyst def add_child_ids(response): - if 'samplesByGuid' in response: - sample_guids_by_individual = defaultdict(list) - for sample in response['samplesByGuid'].values(): - sample_guids_by_individual[sample['individualGuid']].append(sample['sampleGuid']) - if 'igvSamplesByGuid' in response: igv_sample_guids_by_individual = defaultdict(list) for sample in response['igvSamplesByGuid'].values(): @@ -104,8 +99,6 @@ def add_child_ids(response): individual_guids_by_family = defaultdict(list) for individual in response['individualsByGuid'].values(): - if 'samplesByGuid' in response: - individual['sampleGuids'] = sample_guids_by_individual[individual['individualGuid']] if 'igvSamplesByGuid' in response: individual['igvSampleGuids'] = igv_sample_guids_by_individual[individual['individualGuid']] individual_guids_by_family[individual['familyGuid']].append(individual['individualGuid']) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 19088364e6..a16be6a659 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -14,12 +14,12 @@ from seqr.models import SavedVariant, VariantSearchResults, Family, LocusList, LocusListInterval, LocusListGene, \ RnaSeqTpm, PhenotypePrioritization, Project, Dataset, RnaSample, VariantTag, VariantTagType from seqr.utils.gene_utils import get_genes_for_variants -from seqr.utils.xpos_utils import parse_variant_id +from seqr.utils.xpos_utils import parse_variant_id, get_chrom_pos from seqr.views.utils.json_to_orm_utils import create_model_from_json -from seqr.views.utils.orm_to_json_utils import get_json_for_discovery_tags, get_json_for_locus_lists, \ - get_json_for_queryset, get_json_for_rna_seq_outliers, get_json_for_saved_variants_with_tags, \ +from seqr.views.utils.orm_to_json_utils import get_json_for_saved_variants_child_entities, get_json_for_locus_lists, \ + get_json_for_queryset, get_json_for_rna_seq_outliers, get_json_for_saved_variants_with_tags, _get_json_for_families, \ get_json_for_matchmaker_submissions -from seqr.views.utils.permissions_utils import has_case_review_permissions, user_is_analyst +from seqr.views.utils.permissions_utils import has_case_review_permissions, user_is_analyst, get_project_guids_user_can_view from seqr.views.utils.project_context_utils import add_project_tag_types, add_families_context from settings import REDIS_SERVICE_HOSTNAME, REDIS_SERVICE_PORT @@ -59,7 +59,6 @@ def parse_saved_variant_json(variant_json, family_id): 'dataset_type': variant_json.get('dataset_type') or variant_dataset_type(variant_json), 'gene_ids': gene_ids, 'main_transcript': main_transcript, - 'saved_variant_json': variant_json.get('saved_variant_json', {}), } @@ -233,10 +232,6 @@ def reset_cached_search_results(project, reset_index_metadata=False): logger.error("Unable to reset cached search results: {}".format(e)) -def get_variant_key(xpos=None, ref=None, alt=None, genomeVersion=None, **kwargs): - return '{}-{}-{}_{}'.format(xpos, ref, alt, genomeVersion) - - def _requires_transcript_metadata(variant): if isinstance(variant, list): return _requires_transcript_metadata(variant[0]) @@ -355,13 +350,51 @@ def _get_family_has_rna_tpm(family_genes, gene_ids, sample_family_map): return family_tpms -def _add_discovery_tags(variants, discovery_tags): - for variant in variants: - tags = discovery_tags.get(get_variant_key(**variant)) - if tags: - if not variant.get('discoveryTags'): - variant['discoveryTags'] = [] - variant['discoveryTags'] += [tag for tag in tags if tag['savedVariant']['familyGuid'] not in variant['familyGuids']] +def _parse_discovery_tags(variants_by_id, family_guids, user): + discovery_variant_ids = set() + discovery_family_guids = set() + for variant_id, variant in variants_by_id.items(): + if not variant['key']: + continue + discovery_families = set(variant.pop('discoveryFamilies')) - set(variant.get('familyGuids', [])) + if discovery_families: + discovery_variant_ids.add(variant_id) + discovery_family_guids.update(discovery_families) + variant['discoveryTags'] = [] + variant['noAccessDiscoveryFamilies'] = len(discovery_families) + + discovery_families_by_guid = { + f['familyGuid']: f for f in _get_json_for_families(Family.objects.filter( + guid__in=discovery_family_guids, project__guid__in=get_project_guids_user_can_view(user), + ).exclude(guid__in=family_guids)) + } + if not discovery_families_by_guid: + return {} + + saved_variants_by_guid = { + sv['variantGuid']: sv for sv in SavedVariant.objects.filter( + variant_id__in=discovery_variant_ids, family__guid__in=discovery_families_by_guid, + ).values('id', 'variant_id', variantGuid=F('guid'), familyGuid=F('family__guid'), projectGuid=F('family__project__guid')) + } + + saved_variant_id_map = {sv.pop('id'): guid for guid, sv in saved_variants_by_guid.items()} + discovery_tag_json, variant_tag_map = get_json_for_saved_variants_child_entities( + VariantTag, saved_variant_id_map, tag_filter={'variant_tag_type__category': 'CMG Discovery Tags'}) + + tags_by_guid = {} + for tag in discovery_tag_json: + del tag['variantGuids'] + tags_by_guid[tag['tagGuid']] = tag + + for variant_guid, tag_guids in variant_tag_map.items(): + variant = saved_variants_by_guid[variant_guid] + variant_id = variant.pop('variant_id') + variants_by_id[variant_id]['discoveryTags'] += [ + {'savedVariant': variant, **tags_by_guid[tag_guid]} for tag_guid in tag_guids + ] + variants_by_id[variant_id]['noAccessDiscoveryFamilies'] -= 1 + + return {'familiesByGuid': discovery_families_by_guid} def _add_pa_detail(locus_list_gene, locus_list_guid, gene_json): @@ -382,7 +415,7 @@ def get_variants_response(request, saved_variants, response_variants=None, add_a add_locus_list_detail=False, include_individual_gene_scores=True, include_project_name=False, genome_version=None): additional_model_fields = [] if response_variants is None: - additional_model_fields = ['dataset_type', 'saved_variant_json'] + additional_model_fields = ['dataset_type', 'main_transcript', 'xpos_end'] if not genome_version: additional_model_fields.append('family__project__genome_version') response = get_json_for_saved_variants_with_tags(saved_variants, additional_model_fields=additional_model_fields) \ @@ -406,18 +439,11 @@ def get_variants_response(request, saved_variants, response_variants=None, add_a variants, genome_versions={genome_version} if genome_version else {p.genome_version for p in projects}, get_family_genes=include_individual_gene_scores, )) - discovery_tags = None - is_analyst = user_is_analyst(request.user) - if is_analyst: - discovery_tags, discovery_response = get_json_for_discovery_tags(variants, request.user) - response.update(discovery_response) + response.update(_parse_discovery_tags(variants_by_id, family_guids, request.user)) response['locusListsByGuid'] = _add_locus_lists( projects, response['genesById'], add_list_detail=add_locus_list_detail, user=request.user) - if discovery_tags: - _add_discovery_tags(variants, discovery_tags) - response['mmeSubmissionsByGuid'] = _mme_response_context(response['savedVariantsByGuid']) rna_tpm = _set_response_gene_scores(response, response.pop('family_genes'), response['genesById'].keys()) if include_individual_gene_scores else None @@ -434,7 +460,7 @@ def get_variants_response(request, saved_variants, response_variants=None, add_a if add_all_context or request.GET.get(LOAD_FAMILY_CONTEXT_PARAM) == 'true': families = Family.objects.filter(guid__in=family_guids) add_families_context( - response, families, project_guid=project.guid if project else None, user=request.user, is_analyst=is_analyst, + response, families, project_guid=project.guid if project else None, user=request.user, is_analyst=user_is_analyst(request.user), has_case_review_perm=bool(project) and has_case_review_permissions(project, request.user), include_igv=include_igv, ) @@ -454,9 +480,9 @@ def _get_clickhouse_variant_annotations(variants, genome_version): for variant in variants: dataset_type = variant.pop('datasetType') gv = genome_version or variant.pop('familyProjectGenomeVersion') - variant_json = variant.pop('savedVariantJson') + main_transcript = variant.pop('mainTranscript') + xpos_end = variant.pop('xposEnd') variants_by_id[variant['variantId']] = { - **variant_json, **variants_by_id[variant['variantId']], **variant, 'familyGuids': variant['familyGuids'] + variants_by_id[variant['variantId']].get('familyGuids', []), @@ -464,6 +490,24 @@ def _get_clickhouse_variant_annotations(variants, genome_version): } if variant['key']: variant_keys_by_genome_version_dataset_type[gv][dataset_type].add(variant['key']) + else: + chrom, pos = get_chrom_pos(variant['xpos']) + transcripts = {gene_id: [{'geneId': gene_id}] for gene_id in variant['geneIds']} + if main_transcript: + main_gene_id = main_transcript.get('geneId') or variant['geneIds'][0] + transcripts[main_gene_id] = [main_transcript] + variants_by_id[variant['variantId']].update({ + 'chrom': chrom, + 'pos': pos, + 'genomeVersion': gv, + 'transcripts': transcripts, + 'mainTranscriptId': main_transcript.get('transcriptId'), + }) + if variant['svType']: + end_chrom, end = get_chrom_pos(xpos_end) + variants_by_id[variant['variantId']]['end'] = end + if end_chrom != chrom: + variants_by_id[variant['variantId']]['endChrom'] = end_chrom for gv, gv_keys in variant_keys_by_genome_version_dataset_type.items(): for dataset_type, keys in gv_keys.items(): diff --git a/ui/package-lock.json b/ui/package-lock.json index f58320ac31..94c0415ba7 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -4707,23 +4707,23 @@ } }, "node_modules/body-parser": { - "version": "1.20.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", - "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "version": "1.20.5", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", + "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", "dev": true, "dependencies": { - "bytes": "3.1.2", + "bytes": "~3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.13.0", - "raw-body": "2.5.2", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.15.1", + "raw-body": "~2.5.3", "type-is": "~1.6.18", - "unpipe": "1.0.0" + "unpipe": "~1.0.0" }, "engines": { "node": ">= 0.8", @@ -4739,6 +4739,41 @@ "node": ">= 0.8" } }, + "node_modules/body-parser/node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/body-parser/node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/bonjour-service": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/bonjour-service/-/bonjour-service-1.3.0.tgz", @@ -7996,14 +8031,14 @@ } }, "node_modules/express": { - "version": "4.22.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", - "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "version": "4.22.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", + "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", "dev": true, "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "~1.20.3", + "body-parser": "~1.20.5", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", @@ -8022,7 +8057,7 @@ "parseurl": "~1.3.3", "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", - "qs": "~6.14.0", + "qs": "~6.15.1", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", @@ -8071,21 +8106,6 @@ "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "dev": true }, - "node_modules/express/node_modules/qs": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", - "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", - "dev": true, - "dependencies": { - "side-channel": "^1.1.0" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/express/node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -14543,12 +14563,12 @@ } }, "node_modules/qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "version": "6.15.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", + "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", "dev": true, "dependencies": { - "side-channel": "^1.0.6" + "side-channel": "^1.1.0" }, "engines": { "node": ">=0.6" @@ -14645,20 +14665,64 @@ } }, "node_modules/raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", "dev": true, "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" }, "engines": { "node": ">= 0.8" } }, + "node_modules/raw-body/node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/raw-body/node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body/node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/raw-body/node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/react": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react/-/react-17.0.2.tgz", @@ -17303,9 +17367,9 @@ "integrity": "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==" }, "node_modules/tmp": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", - "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==", + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", + "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", "dev": true, "engines": { "node": ">=14.14" @@ -22720,23 +22784,23 @@ "dev": true }, "body-parser": { - "version": "1.20.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", - "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "version": "1.20.5", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", + "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", "dev": true, "requires": { - "bytes": "3.1.2", + "bytes": "~3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.13.0", - "raw-body": "2.5.2", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.15.1", + "raw-body": "~2.5.3", "type-is": "~1.6.18", - "unpipe": "1.0.0" + "unpipe": "~1.0.0" }, "dependencies": { "depd": { @@ -22744,6 +22808,31 @@ "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", "dev": true + }, + "http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, + "requires": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true } } }, @@ -25408,14 +25497,14 @@ } }, "express": { - "version": "4.22.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", - "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "version": "4.22.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", + "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", "dev": true, "requires": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "~1.20.3", + "body-parser": "~1.20.5", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", @@ -25434,7 +25523,7 @@ "parseurl": "~1.3.3", "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", - "qs": "~6.14.0", + "qs": "~6.15.1", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", @@ -25470,15 +25559,6 @@ "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "dev": true }, - "qs": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", - "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", - "dev": true, - "requires": { - "side-channel": "^1.1.0" - } - }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -30647,12 +30727,12 @@ } }, "qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "version": "6.15.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", + "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", "dev": true, "requires": { - "side-channel": "^1.0.6" + "side-channel": "^1.1.0" } }, "query-string": { @@ -30731,15 +30811,48 @@ "dev": true }, "raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", "dev": true, "requires": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "dependencies": { + "depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true + }, + "http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, + "requires": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true + } } }, "react": { @@ -32902,9 +33015,9 @@ "integrity": "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==" }, "tmp": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", - "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==", + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", + "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", "dev": true }, "tmpl": { diff --git a/ui/pages/Public/components/Faq.jsx b/ui/pages/Public/components/Faq.jsx index 261c4f8f4f..d367dabd5f 100644 --- a/ui/pages/Public/components/Faq.jsx +++ b/ui/pages/Public/components/Faq.jsx @@ -2,10 +2,10 @@ import PropTypes from 'prop-types' import React from 'react' -import { Header, Segment, List, Icon } from 'semantic-ui-react' +import { Header, Segment, List, Icon, Accordion } from 'semantic-ui-react' import { WORKSPACE_REQUIREMENTS } from 'shared/components/panel/LoadWorkspaceDataForm' -import { ActiveDisabledNavLink } from 'shared/components/StyledComponents' +import { ActiveDisabledNavLink, InlineHeader } from 'shared/components/StyledComponents' import { VCF_DOCUMENTATION_URL } from 'shared/utils/constants' import { SeqrAvailability } from './LandingPage' @@ -519,6 +519,33 @@ const FAQS = [ ), }, + }, { + [ENGLISH]: { + header: 'Q. Can I get assistance with processing my data?', + content: ( +
+ Yes - If you need support processing your data for seqr, such as generating a joint-called VCF from CRAM files + or reprocessing data using DRAGEN, the Broad Institute’s Data Science Services group offers fee-for-service + support. These services are designed to help prepare your data for successful ingestion into seqr and are + available to both internal Broad cohorts and external research groups. To discuss project scope, timelines, + and pricing, please reach out to bclservices@broadinstitute.org. +
+ ), + }, + [SPANISH]: { + header: 'P. ¿Puedo recibir asistencia para procesar mis datos?', + content: ( +
+ Sí. Si necesita asistencia para el procesamiento de sus datos destinados a seqr - como la generación de un + archivo VCF de llamada conjunta a partir de archivos CRAM, o el reprocesamiento de datos mediante DRAGEN - el + grupo de Servicios de Ciencia de Datos del Broad Institute ofrece soporte bajo la modalidad de pago por + servicio. Estos servicios están diseñados para ayudarle a preparar sus datos y asegurar su correcta ingesta en + seqr, y se encuentran a disposición tanto de las cohortes internas del Broad como de grupos de investigación + externos. Para conversar sobre el alcance del proyecto, los plazos y las tarifas, por favor póngase en + contacto con bclservices@broadinstitute.org. +
+ ), + }, }, { [ENGLISH]: { header: 'Q. Who has access to my data in seqr?', @@ -608,6 +635,15 @@ const FAQS = [ }, ] +const LANGUAGE_PANELS = [ENGLISH, SPANISH].reduce((acc, language) => ({ + ...acc, + [language]: FAQS.map(faq => ({ + key: faq[language].header, + title: { content: }, + content: { content: {faq[language].content} }, + })), +}), {}) + const LANGUAGES = [{ path: '', text: 'English' }, { path: SPANISH, text: 'Español' }] const FaqPages = ({ match }) => ( @@ -620,13 +656,7 @@ const FaqPages = ({ match }) => ( {text} ))} /> - {FAQS.map((config) => { - const { header, content } = config[match.params.language || ENGLISH] - return [ -
, - content, - ] - })} + ) diff --git a/ui/pages/Report/components/VariantMetadata.jsx b/ui/pages/Report/components/VariantMetadata.jsx index b077b70c36..a06596f36f 100644 --- a/ui/pages/Report/components/VariantMetadata.jsx +++ b/ui/pages/Report/components/VariantMetadata.jsx @@ -1,7 +1,7 @@ import React from 'react' import LoadReportTable from 'shared/components/table/LoadReportTable' -import { clinvarSignificance, VARIANT_METADATA_COLUMNS } from 'shared/utils/constants' +import { VARIANT_METADATA_COLUMNS } from 'shared/utils/constants' const VIEW_ALL_PAGES = [ { name: 'GREGoR', downloadName: 'GREGoR', path: 'gregor' }, @@ -14,7 +14,7 @@ const COLUMNS = [ { name: 'variant_type' }, { name: 'allele_balance_or_heteroplasmy_percentage' }, { name: 'Clinvar allele ID', format: ({ clinvar }) => clinvar?.alleleId }, - { name: 'ClinVar Clinical Significance', format: ({ clinvar }) => clinvarSignificance(clinvar).pathogenicity }, + { name: 'ClinVar Clinical Significance', format: ({ clinvar }) => clinvar?.pathogenicity }, { name: 'ClinVar gold star', format: ({ clinvar }) => clinvar?.goldStars }, { name: 'known_condition_name' }, { name: 'condition_id' }, diff --git a/ui/pages/Search/VariantLookup.jsx b/ui/pages/Search/VariantLookup.jsx index 4b9145a4bf..83152944c0 100644 --- a/ui/pages/Search/VariantLookup.jsx +++ b/ui/pages/Search/VariantLookup.jsx @@ -7,6 +7,7 @@ import { RECEIVE_DATA } from 'redux/utils/reducerUtils' import { navigateSavedHashedSearch } from 'redux/rootReducer' import { getVlmEnabled } from 'redux/selectors' import { QueryParamsEditor } from 'shared/components/QueryParamEditor' +import { VerticalSpacer } from 'shared/components/Spacers' import StateDataLoader from 'shared/components/StateDataLoader' import SendEmailButton from 'shared/components/buttons/SendEmailButton' import FormWrapper from 'shared/components/form/FormWrapper' @@ -193,6 +194,35 @@ InternalFamily.propTypes = { showReads: PropTypes.object, } +const ExternalFamily = ({ familyGuid, variant }) => { + const { discoveryTagFamilies = [], excludedTagFamilies = [] } = variant + const tags = [] + if (discoveryTagFamilies.includes(familyGuid)) { + tags.push({ color: 'teal', content: 'Discovery Variant' }) + } + if (excludedTagFamilies.includes(familyGuid)) { + tags.push({ color: 'grey', content: 'Excluded' }) + } + return ( + + {tags.length > 0 && ( + + {tags.map(tag => + )} + + + + + ) +} + +ExternalFamily.propTypes = { + familyGuid: PropTypes.string, + variant: PropTypes.object, +} + const BaseLookupVariant = ({ variant, familiesByContactEmail, vlmDefaultContactEmails }) => { const { internal, disabled, ...familiesByContact } = familiesByContactEmail return ( @@ -211,18 +241,14 @@ const BaseLookupVariant = ({ variant, familiesByContactEmail, vlmDefaultContactE > {families.map(familyGuid => ( - - - - - + ))} ))} {(disabled || []).map(familyGuid => ( - + ))} diff --git a/ui/shared/components/panel/family/FamilyReads.jsx b/ui/shared/components/panel/family/FamilyReads.jsx index bba09bd91f..70d6a41a55 100644 --- a/ui/shared/components/panel/family/FamilyReads.jsx +++ b/ui/shared/components/panel/family/FamilyReads.jsx @@ -196,11 +196,11 @@ const applyUserTrackSettings = (tracks, options) => tracks.map(track => ({ } : {}, })) -const getVariantLocus = (variant, project) => { +const getVariantLocus = (variant) => { const size = variant.end && variant.end - variant.pos return getLocus( variant.chrom, - (variant.genomeVersion !== project.genomeVersion && variant.liftedOverPos) ? variant.liftedOverPos : variant.pos, + variant.pos, size ? Math.max(Math.round(size / 2), MIN_LOCUS_RANGE_SIZE) : MIN_LOCUS_RANGE_SIZE, size, ) @@ -309,7 +309,7 @@ class FamilyReads extends React.PureComponent { this.setState({ openFamily: familyGuid, sampleTypes, - locus: variant && getVariantLocus(variant, this.getProjectForFamily(familyGuid)), + locus: variant && getVariantLocus(variant), }) } @@ -416,7 +416,7 @@ class FamilyReads extends React.PureComponent { const project = openFamily && this.getProjectForFamily(openFamily) const geneLocus = project && variant && getGeneLocus(variant, genesById, project) const locusOptions = [ - { text: 'Variant', value: geneLocus && getVariantLocus(variant, project) }, + { text: 'Variant', value: geneLocus && getVariantLocus(variant) }, { text: 'Gene', value: geneLocus }, { text: 'Splice Outlier', diff --git a/ui/shared/components/panel/family/constants.js b/ui/shared/components/panel/family/constants.js index 0b74b27de4..37c4521d61 100644 --- a/ui/shared/components/panel/family/constants.js +++ b/ui/shared/components/panel/family/constants.js @@ -80,7 +80,7 @@ export const IGV_OPTIONS = { const REFERENCE_URLS = [ { key: 'fastaURL', - baseUrl: 'https://igv-genepattern-org.s3.amazonaws.com/genomes/seq', + baseUrl: 'https://s3.amazonaws.com/igv.broadinstitute.org/genomes/seq', path: { 37: 'hg19/hg19.fasta', 38: 'hg38/hg38.fa', @@ -96,10 +96,10 @@ const REFERENCE_URLS = [ }, { key: 'aliasURL', - baseUrl: undefined, + baseUrl: 'https://igv.org/genomes/data', path: { - 37: 'https://igv.org/genomes/data/hg19/hg19_alias.tab', - 38: 'https://igv-genepattern-org.s3.amazonaws.com/genomes/hg38/hg38_alias.tab', + 37: 'hg19/hg19_alias.tab', + 38: 'hg38/hg38_alias.tab', }, }, ] diff --git a/ui/shared/components/panel/variants/Annotations.jsx b/ui/shared/components/panel/variants/Annotations.jsx index fbb99e52fd..c41f434f0c 100644 --- a/ui/shared/components/panel/variants/Annotations.jsx +++ b/ui/shared/components/panel/variants/Annotations.jsx @@ -449,11 +449,9 @@ const svSizeDisplay = (size) => { return `${(size / 1000000).toFixed(2) / 1}Mb` } -const getLofDetails = ({ isLofNagnag, lofFilters, lofFilter, lofFlags, lof }) => { - const isNagnag = isLofNagnag || lofFlags === 'NAGNAG_SITE' - const filters = lofFilters || (lof === 'LC' && lofFilter && lofFilter.split(/&|,/g)) - return (filters || isNagnag) ? [ - ...(filters ? [...new Set(filters)] : []).map((lofFilterKey) => { +const getLofDetails = ({ isLofNagnag, lofFilters }) => ( + (lofFilters || isLofNagnag) ? [ + ...(lofFilters ? [...new Set(lofFilters)] : []).map((lofFilterKey) => { const filter = LOF_FILTER_MAP[lofFilterKey] || { message: lofFilterKey } return (
@@ -463,7 +461,7 @@ const getLofDetails = ({ isLofNagnag, lofFilters, lofFilter, lofFlags, lof }) =>
) }), - isNagnag ? ( + isLofNagnag ? (
LOFTEE: NAGNAG site
@@ -471,7 +469,7 @@ const getLofDetails = ({ isLofNagnag, lofFilters, lofFilter, lofFlags, lof }) =>
) : null, ] : null -} +) // Adapted from https://github.com/ImperialCardioGenetics/UTRannotator/blob/master/README.md#the-detailed-annotation-for-each-consequence const UTR_ANNOTATOR_DESCRIPTIONS = { @@ -533,7 +531,7 @@ const Annotations = React.memo(({ variant, mainGeneId, showMainGene, transcripts endChrom, CAID, } = variant const mainTranscript = getVariantMainTranscript(variant) - const lofDetails = getLofDetails(mainTranscript.loftee || mainTranscript) + const lofDetails = getLofDetails(mainTranscript.loftee || {}) const transcriptPopupProps = mainTranscript.transcriptId && { content: , diff --git a/ui/shared/components/panel/variants/Frequencies.jsx b/ui/shared/components/panel/variants/Frequencies.jsx index ad508e295f..19b6a862ac 100644 --- a/ui/shared/components/panel/variants/Frequencies.jsx +++ b/ui/shared/components/panel/variants/Frequencies.jsx @@ -175,12 +175,11 @@ FreqSummary.propTypes = { const getGenePath = ({ variant }) => `gene/${getVariantMainGeneId(variant)}` -const gnomadLink = ({ fieldTitle, esVersion, variant, ...props }) => { - const isEs = !(variant || {}).populations?.seqr +const gnomadLink = ({ fieldTitle, variant, ...props }) => { const [prefix, detail] = fieldTitle.split(' ') return ( - +   {detail} @@ -191,15 +190,14 @@ gnomadLink.propTypes = { fieldTitle: PropTypes.string, } -const gnomadAnWarning = ({ ac, an }, { fieldTitle, maxAN, variant }) => { - const isEs = !(variant || {}).populations?.seqr - return (!isEs && ac && an < (maxAN / 2)) ? ( +const gnomadAnWarning = ({ ac, an }, { fieldTitle, maxAN }) => ( + (ac && an < (maxAN / 2)) ? ( } content={`This variant is covered in fewer than 50% of individuals in ${fieldTitle}. This may indicate a low-quality site.`} /> ) : null -} +) const GNOMAD_URL_INFO = { urls: { [GENOME_VERSION_37]: 'gnomad.broadinstitute.org', [GENOME_VERSION_38]: 'gnomad.broadinstitute.org' }, @@ -242,15 +240,11 @@ GlobalAcSampleTypeSummary.propTypes = { const HOM_SECTION = 'Homoplasmy' const HET_SECTION = 'Heteroplasmy' -const SV_CALLSET_POP = { field: 'sv_callset', fieldTitle: 'This Callset', acDisplay: 'AC', helpMessage: SV_CALLSET_CRITERIA_MESSAGE } -const CALLSET_POP = { field: 'callset', fieldTitle: 'This Callset', acDisplay: 'AC' } -const SEQR_POP = { ...CALLSET_POP, field: 'seqr', fieldTitle: 'seqr' } +const SEQR_POP = { field: 'seqr', fieldTitle: 'seqr', acDisplay: 'AC' } const POPULATIONS = [ - SV_CALLSET_POP, - { ...SV_CALLSET_POP, field: 'sv_seqr', fieldTitle: 'seqr' }, + { field: 'sv_seqr', fieldTitle: 'seqr', acDisplay: 'AC', helpMessage: SV_CALLSET_CRITERIA_MESSAGE }, { ...SEQR_POP, field: 'sv_seqr_affected', fieldTitle: 'seqr affected' }, - CALLSET_POP, SEQR_POP, { ...SEQR_POP, field: 'seqr_affected', fieldTitle: 'seqr affected' }, { @@ -259,7 +253,6 @@ const POPULATIONS = [ titleContainer: gnomadLink, warningContainer: gnomadAnWarning, maxAN: 730947 * 2, // From https://gnomad.broadinstitute.org/stats - esVersion: 'v2', conditionalQueryParams: populations => (populations.seqr ? GNOMAD_URL_INFO.queryParams : { [GENOME_VERSION_37]: 'dataset=gnomad_r2_1' }), ...GNOMAD_URL_INFO, }, @@ -269,7 +262,6 @@ const POPULATIONS = [ titleContainer: gnomadLink, warningContainer: gnomadAnWarning, maxAN: 76215 * 2, // From https://gnomad.broadinstitute.org/stats - esVersion: 'v3', conditionalQueryParams: populations => (populations.seqr ? GNOMAD_URL_INFO.queryParams : { [GENOME_VERSION_38]: 'dataset=gnomad_r3' }), precision: 3, ...GNOMAD_URL_INFO, @@ -293,10 +285,8 @@ const POPULATIONS = [ }, ] -const CALLSET_HET_POP = { - field: 'callset_heteroplasmy', - fieldTitle: 'This Callset', - acDisplay: 'AC', +const SEQR_HET_POP = { + ...SEQR_POP, titleContainer: sectionTitle, section: HET_SECTION, } @@ -308,20 +298,14 @@ const SEQR_HOM_POP = { } const MITO_POPULATIONS = [ - { - ...CALLSET_POP, - titleContainer: sectionTitle, - section: HOM_SECTION, - }, { ...SEQR_POP, titleContainer: sectionTitle, section: HOM_SECTION, }, { ...SEQR_HOM_POP, field: 'seqr_affected', fieldTitle: 'seqr affected' }, - CALLSET_HET_POP, - { ...CALLSET_HET_POP, field: 'seqr_heteroplasmy', fieldTitle: 'seqr' }, - { ...CALLSET_HET_POP, field: 'seqr_heteroplasmy_affected', fieldTitle: 'seqr affected' }, + { ...SEQR_HET_POP, field: 'seqr_heteroplasmy', fieldTitle: 'seqr' }, + { ...SEQR_HET_POP, field: 'seqr_heteroplasmy_affected', fieldTitle: 'seqr affected' }, { field: 'gnomad_mito', fieldTitle: 'gnomAD mito', @@ -388,8 +372,7 @@ const getValueDisplay = (pop, valueField, precision) => (valueField === 'ac' ? const Frequencies = React.memo(({ variant, totalSampleCounts }) => { const { populations = {}, svType } = variant - const callsetHetPop = populations.callset_heteroplasmy || populations.seqr_heteroplasmy - const isMito = callsetHetPop && callsetHetPop.ac !== null && callsetHetPop.ac !== undefined + const isMito = !!populations.seqr_heteroplasmy const popConfigs = isMito ? MITO_POPULATIONS : POPULATIONS let datasetType = isMito ? DATASET_TYPE_MITO_CALLS : DATASET_TYPE_SNV_INDEL_CALLS datasetType = svType ? DATASET_TYPE_SV_CALLS : datasetType diff --git a/ui/shared/components/panel/variants/Pathogenicity.jsx b/ui/shared/components/panel/variants/Pathogenicity.jsx index f84517a640..fcf4b9fa9a 100644 --- a/ui/shared/components/panel/variants/Pathogenicity.jsx +++ b/ui/shared/components/panel/variants/Pathogenicity.jsx @@ -3,10 +3,10 @@ import PropTypes from 'prop-types' import { connect } from 'react-redux' import styled from 'styled-components' import { Label, Icon, Popup, List, ListItem } from 'semantic-ui-react' -import { HorizontalSpacer, VerticalSpacer } from 'shared/components/Spacers' +import { HorizontalSpacer } from 'shared/components/Spacers' import { getUser, getFamiliesByGuid, getProjectsByGuid } from 'redux/selectors' -import { clinvarSignificance, clinvarColor, getPermissionedHgmdClass } from '../../../utils/constants' +import { clinvarColor, getPermissionedHgmdClass } from '../../../utils/constants' import { snakecaseToTitlecase } from '../../../utils/stringUtils' const StarsContainer = styled.span` @@ -21,7 +21,7 @@ const StarIcon = styled(Icon).attrs({ name: 'star' })` const HGMD_CLASS_NAMES = { DM: 'Disease Causing (DM)', 'DM?': 'Disease Causing? (DM?)', - FPV: 'Frameshift or truncating variant (FTV)', + R: 'Removed', FP: 'In vitro/laboratory or in vivo functional polymorphism (FP)', DFP: 'Disease-associated polymorphism with additional supporting functional evidence (DFP)', DP: 'Disease-associated polymorphism (DP)', @@ -69,12 +69,6 @@ PathogenicityLink.propTypes = { popup: PropTypes.object, } -const clinvarUrl = (clinvar) => { - const baseUrl = 'http://www.ncbi.nlm.nih.gov/clinvar' - const variantPath = clinvar.alleleId ? `?term=${clinvar.alleleId}[alleleid]` : `/variation/${clinvar.variationId}` - return baseUrl + variantPath -} - const clinvarLabel = (pathogenicity, assertions, conflictingPathogenicities) => { let label = snakecaseToTitlecase(pathogenicity) if (conflictingPathogenicities && conflictingPathogenicities.length) { @@ -89,42 +83,27 @@ const clinvarLabel = (pathogenicity, assertions, conflictingPathogenicities) => return label } -const clinvarPopup = (clinvar) => { - const lastUpdated = ( -
{clinvar.version && `Last Updated: ${new Date(clinvar.version).toLocaleDateString()}`}
- ) - const conditions = clinvar.conditions && ( +const clinvarPopup = ({ conditions }) => ( + conditions ? (
Conditions: - {[...new Set(clinvar.conditions)].map(condition => ( + {[...new Set(conditions)].map(condition => ( {condition} ))}
- ) - return ( -
- {lastUpdated} - {conditions && ( -
- - {conditions} -
- )} -
- ) -} + ) : null +) const Pathogenicity = React.memo(({ variant, showHgmd }) => { const clinvar = variant.clinvar || {} const pathogenicity = [] - if ((clinvar.clinicalSignificance || clinvar.pathogenicity) && (clinvar.variationId || clinvar.alleleId)) { - const { pathogenicity: clinvarPathogenicity, assertions, severity } = clinvarSignificance(clinvar) + if (clinvar.pathogenicity && clinvar.alleleId) { pathogenicity.push(['ClinVar', { - label: clinvarLabel(clinvarPathogenicity, assertions, clinvar.conflictingPathogenicities), - color: clinvarColor(severity, 'red', 'orange', 'green'), - href: clinvarUrl(clinvar), + label: clinvarLabel(clinvar.pathogenicity, clinvar.assertions, clinvar.conflictingPathogenicities), + color: clinvarColor(clinvar, 'red', 'orange', 'green'), + href: `http://www.ncbi.nlm.nih.gov/clinvar?term=${clinvar.alleleId}[alleleid]`, goldStars: clinvar.goldStars, popup: clinvarPopup(clinvar), submitters: clinvar.submitters, diff --git a/ui/shared/components/panel/variants/VariantIndividuals.jsx b/ui/shared/components/panel/variants/VariantIndividuals.jsx index 35ad3bdaff..72c2a29aa4 100644 --- a/ui/shared/components/panel/variants/VariantIndividuals.jsx +++ b/ui/shared/components/panel/variants/VariantIndividuals.jsx @@ -202,18 +202,10 @@ Alleles.propTypes = { const GENOTYPE_DETAILS = [ { title: 'Sample Type', field: 'sampleType' }, - { - title: 'Raw Alt. Alleles', - variantField: 'originalAltAlleles', - format: val => (val || []).join(', '), - shouldHide: (val, variant) => (val || []).length < 1 || ((val || []).length === 1 && val[0] === variant.alt), - }, - { title: 'Allelic Depth', field: 'ad' }, { title: 'Read Depth', field: 'dp' }, { title: 'Genotype Quality', field: 'gq' }, { title: 'Allelic Balance', field: 'ab', format: val => val && val.toPrecision(2) }, - { title: 'Filter', field: 'filters', variantField: 'genotypeFilters', shouldHide: val => (val || []).length < 1 }, - { title: 'Phred Likelihoods', field: 'pl' }, + { title: 'Filter', field: 'filters' }, { title: 'Quality Score', field: 'qs' }, { title: 'Mitochondrial Copy Number', @@ -236,10 +228,10 @@ const SV_GENOTYPE_DETAILS = [ }, ] -const formattedGenotypeDetails = (details, genotype, variant, genesById) => details.map( - ({ shouldHide, title, field, variantField, format, comment }) => { - const value = genotype[field] || variant[variantField] - return value && !(shouldHide && shouldHide(value, variant)) ? ( +const formattedGenotypeDetails = (details, genotype, genesById) => details.map( + ({ title, field, format, comment }) => { + const value = genotype[field] + return value ? (
{`${title}: `} {format ? format(value, genesById) : value} @@ -249,9 +241,9 @@ const formattedGenotypeDetails = (details, genotype, variant, genesById) => deta }, ).filter(val => val) -const genotypeDetails = (genotype, variant, genesById) => { - const details = formattedGenotypeDetails(GENOTYPE_DETAILS, genotype, variant) - const svDetails = formattedGenotypeDetails(SV_GENOTYPE_DETAILS, genotype, variant, genesById) +const genotypeDetails = (genotype, genesById) => { + const details = formattedGenotypeDetails(GENOTYPE_DETAILS, genotype) + const svDetails = formattedGenotypeDetails(SV_GENOTYPE_DETAILS, genotype, genesById) if (svDetails.length < 1) { return details } @@ -393,7 +385,7 @@ const GenotypeQuality = ({ genotype, variant, showSampleType }) => { const showSecondaryQuality = !variant.svType && genotype.numAlt >= 0 const secondaryQuality = genotype.ab || genotype.hl const quality = Number.isInteger(genotype.gq) ? genotype.gq : genotype.qs - const filters = genotype.filters?.join(', ') || variant.genotypeFilters + const filters = genotype.filters?.join(', ') return (
@@ -442,8 +434,8 @@ const Genotype = React.memo(({ variant, individual, isCompoundHet, genesById }) const details = genotypes.flatMap((genotype, index) => ( index === 0 ? - [genotypeDetails(genotype, variant, genesById)] : - [, genotypeDetails(genotype, variant, genesById)] + [genotypeDetails(genotype, genesById)] : + [, genotypeDetails(genotype, genesById)] )) const content = ( @@ -474,14 +466,31 @@ Genotype.propTypes = { genesById: PropTypes.object, } -const INDIVIDUAL_DETAIL_FIELDS = [INDIVIDUAL_FIELD_FEATURES] +const INDIVIDUAL_FIELD_IS_SOLVED = 'isSolved' +const INDIVIDUAL_FIELD_DISEASE = 'disease' +const INDIVIDUAL_DETAIL_FIELDS = [INDIVIDUAL_FIELD_DISEASE, INDIVIDUAL_FIELD_IS_SOLVED, INDIVIDUAL_FIELD_FEATURES] const VARIANT_INDIVIDUAL_DETAIL_FIELDS = [ INDIVIDUAL_FIELD_FILTER_FLAGS, INDIVIDUAL_FIELD_POP_FILTERS, ...INDIVIDUAL_DETAIL_FIELDS, ] const SV_INDIVIDUAL_DETAIL_FIELDS = [INDIVIDUAL_FIELD_SV_FLAGS, ...INDIVIDUAL_DETAIL_FIELDS] +const VARIANT_INDIVIDUAL_FIELD_LOOKUP = { + ...INDIVIDUAL_FIELD_LOOKUP, + [INDIVIDUAL_FIELD_IS_SOLVED]: { + fieldDisplay: isSolved => (isSolved ?
diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index 908acc8294..1b8c22e55c 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -750,27 +750,27 @@ export const INDIVIDUAL_FIELD_LOOKUP = { // CLINVAR -const CLINVAR_DEFAULT_PATHOGENICITY = 'no_pathogenic_assertion' -const CLINVAR_MAX_RISK_PATHOGENICITY = 'established_risk_allele' -const CLINVAR_MIN_RISK_PATHOGENICITY = 'likely_risk_allele' +const CLINVAR_DEFAULT_PATHOGENICITY = 'No_pathogenic_assertion' +const CLINVAR_MAX_RISK_PATHOGENICITY = 'Established_risk_allele' +const CLINVAR_MIN_RISK_PATHOGENICITY = 'Likely_risk_allele' const CLINVAR_PATHOGENICITIES = [ - 'pathogenic', - 'pathogenic/likely_pathogenic', - 'pathogenic/likely_pathogenic/established_risk_allele', - 'pathogenic/likely_pathogenic/likely_risk_allele', - 'pathogenic/likely_risk_allele', - 'likely_pathogenic', - 'likely_pathogenic/likely_risk_allele', + 'Pathogenic', + 'Pathogenic/Likely_pathogenic', + 'Pathogenic/Likely_pathogenic/Established_risk_allele', + 'Pathogenic/Likely_pathogenic/Likely_risk_allele', + 'Pathogenic/Likely_risk_allele', + 'Likely_pathogenic', + 'Likely_pathogenic/Likely_risk_allele', CLINVAR_MAX_RISK_PATHOGENICITY, CLINVAR_MIN_RISK_PATHOGENICITY, - 'conflicting_interpretations_of_pathogenicity', - 'uncertain_risk_allele', - 'uncertain_significance/uncertain_risk_allele', - 'uncertain_significance', + 'Conflicting_classifications_of_pathogenicity', + 'Uncertain_risk_allele', + 'Uncertain_significance/Uncertain_risk_allele', + 'Uncertain_significance', CLINVAR_DEFAULT_PATHOGENICITY, - 'likely_benign', - 'benign/likely_benign', - 'benign', + 'Likely_benign', + 'Benign/Likely_benign', + 'Benign', ].reverse().reduce((acc, path, i) => ({ ...acc, [path]: i }), {}) const HGMD_SEVERITY = { @@ -1281,7 +1281,7 @@ const SORT_BY_PRIORITIZED_GENE = 'PRIORITIZED_GENE' const SORT_BY_PROTEIN_CONSQ = 'PROTEIN_CONSEQUENCE' const SORT_BY_GNOMAD_GENOMES = 'GNOMAD' const SORT_BY_GNOMAD_EXOMES = 'GNOMAD_EXOMES' -const SORT_BY_CALLSET_AF = 'CALLSET_AF' +const SORT_BY_SEQR_AC = 'SEQR_AC' const SORT_BY_CONSTRAINT = 'CONSTRAINT' const SORT_BY_CADD = 'CADD' const SORT_BY_REVEL = 'REVEL' @@ -1298,27 +1298,8 @@ export const getPermissionedHgmdClass = (variant, user, familiesByGuid, projectB familyGuid => projectByGuid[familiesByGuid[familyGuid].projectGuid].enableHgmd, )) && variant.hgmd && variant.hgmd.class -export const clinvarSignificance = (clinvar) => { - let { pathogenicity, assertions } = clinvar || {} - const { clinicalSignificance } = clinvar || {} - if (clinicalSignificance && !pathogenicity) { - [pathogenicity, ...assertions] = clinicalSignificance.split(/[,|]/) - if (pathogenicity === 'Pathogenic/Likely_pathogenic/Pathogenic') { - pathogenicity = 'Pathogenic/Likely_pathogenic' - } else if (pathogenicity === 'Pathogenic/Pathogenic') { - pathogenicity = 'Pathogenic' - } - if (!(pathogenicity.replace(' ', '_').toLowerCase() in CLINVAR_PATHOGENICITIES)) { - assertions = [pathogenicity, ...assertions] - pathogenicity = CLINVAR_DEFAULT_PATHOGENICITY - } - assertions = assertions.map(a => a.replace(/^_/, '')) - } - - return { pathogenicity, assertions, severity: CLINVAR_PATHOGENICITIES[pathogenicity?.replace(' ', '_').toLowerCase()] } -} - -export const clinvarColor = (severity, pathColor, riskColor, benignColor) => { +export const clinvarColor = (clinvar, pathColor, riskColor, benignColor) => { + const severity = CLINVAR_PATHOGENICITIES[clinvar?.pathogenicity] if (severity > CLINVAR_PATHOGENICITIES[CLINVAR_MAX_RISK_PATHOGENICITY]) { return pathColor } @@ -1332,10 +1313,10 @@ export const clinvarColor = (severity, pathColor, riskColor, benignColor) => { } const clinsigSeverity = (variant, user, familiesByGuid, projectByGuid) => { - const { pathogenicity, severity } = clinvarSignificance(variant.clinvar) + const { pathogenicity } = variant.clinvar || {} const hgmdSignificance = getPermissionedHgmdClass(variant, user, familiesByGuid, projectByGuid) if (!pathogenicity && !hgmdSignificance) return -10 - const clinvarSeverity = pathogenicity ? severity + 1 : 0.1 + const clinvarSeverity = pathogenicity ? CLINVAR_PATHOGENICITIES[pathogenicity] + 1 : 0.1 const hgmdSeverity = HGMD_SEVERITY[hgmdSignificance] || 0 return clinvarSeverity + hgmdSeverity } @@ -1409,7 +1390,7 @@ const VARIANT_SORT_OPTONS = [ }, { value: SORT_BY_GNOMAD_GENOMES, text: 'gnomAD Genomes Frequency', comparator: populationComparator('gnomad_genomes') }, { value: SORT_BY_GNOMAD_EXOMES, text: 'gnomAD Exomes Frequency', comparator: populationComparator('gnomad_exomes') }, - { value: SORT_BY_CALLSET_AF, text: 'Callset AF', comparator: populationComparator('callset') }, + { value: SORT_BY_SEQR_AC, text: 'seqr AC', comparator: (a, b) => a.populations?.seqr?.ac - b.populations?.seqr?.ac }, { value: SORT_BY_CADD, text: 'CADD', comparator: predictionComparator('cadd') }, { value: SORT_BY_REVEL, text: 'REVEL', comparator: predictionComparator('revel') }, { value: SORT_BY_EIGEN, text: 'Eigen', comparator: predictionComparator('eigen') }, @@ -1582,7 +1563,6 @@ export const ORDERED_PREDICTOR_FIELDS = [ { field: 'mut_pred', thresholds: [0.0101, 0.392, 0.737, 0.829, 0.932], fieldTitle: 'MutPred', citation: CLINGEN_CITATION }, { field: 'primate_ai', group: MISSENSE_IN_SILICO_GROUP, thresholds: [undefined, 0.484, 0.79, 0.867, undefined], fieldTitle: 'PrimateAI', citation: CLINGEN_CITATION }, { field: 'eigen', group: CODING_IN_SILICO_GROUP, thresholds: [undefined, undefined, 1, 2, undefined], max: 99 }, - { field: 'dann', displayOnly: true, thresholds: [undefined, undefined, 0.93, 0.96, undefined] }, { field: 'strvctvre', group: SV_IN_SILICO_GROUP, thresholds: [undefined, undefined, 0.5, 0.75, undefined] }, { field: 'polyphen', group: MISSENSE_IN_SILICO_GROUP, thresholds: [undefined, 0.114, 0.978, 0.999, undefined], fieldTitle: 'PolyPhen', citation: CLINGEN_CITATION }, { field: 'sift', reverseThresholds: true, thresholds: [undefined, 0, 0.002, 0.081, undefined], group: MISSENSE_IN_SILICO_GROUP, fieldTitle: 'SIFT', citation: CLINGEN_CITATION }, @@ -1596,7 +1576,7 @@ export const ORDERED_PREDICTOR_FIELDS = [ thresholds: [undefined, undefined, 2.18, 4, undefined], citation: CLINGEN_CITATION, }, - { field: 'haplogroup_defining', indicatorMap: { Y: { color: 'green', value: '' }, true: { color: 'green', value: '' } } }, + { field: 'haplogroup_defining', indicatorMap: { true: { color: 'green', value: '' } } }, { field: 'mitotip', indicatorMap: MITOTIP_MAP, fieldTitle: 'MitoTIP' }, { field: 'hmtvar', thresholds: [undefined, undefined, 0.35, 0.35, undefined], fieldTitle: 'HmtVar' }, { field: 'mlc', thresholds: [undefined, 0.5, 0.5, 0.75, undefined], fieldTitle: 'MLC' }, @@ -1628,7 +1608,7 @@ export const predictionFieldValue = ( ) } - return indicatorMap && (indicatorMap[value[0]] || indicatorMap[value]) + return indicatorMap && indicatorMap[value] } export const predictorColorRanges = (thresholds, { citation, reverseThresholds, thresholdMap, absValue }) => (
@@ -1727,7 +1707,7 @@ export const VARIANT_EXPORT_DATA = [ { header: 'alt' }, { header: 'gene', getVal: getVariantGene }, { header: 'worst_consequence', getVal: variant => getVariantMainTranscript(variant).majorConsequence }, - { header: 'callset_freq', getVal: variant => getPopAf('callset')(variant) || getPopAf('seqr')(variant) }, + { header: 'callset_freq', getVal: variant => getPopAf('seqr')(variant) }, { header: 'exac_freq', getVal: getPopAf('exac') }, { header: 'gnomad_genomes_freq', getVal: getPopAf('gnomad_genomes') }, { header: 'gnomad_exomes_freq', getVal: getPopAf('gnomad_exomes') }, @@ -1743,7 +1723,7 @@ export const VARIANT_EXPORT_DATA = [ { header: 'rsid', getVal: variant => variant.rsid }, { header: 'hgvsc', getVal: variant => getVariantMainTranscript(variant).hgvsc }, { header: 'hgvsp', getVal: variant => getVariantMainTranscript(variant).hgvsp }, - { header: 'clinvar_clinical_significance', getVal: variant => (variant.clinvar || {}).clinicalSignificance || (variant.clinvar || {}).pathogenicity }, + { header: 'clinvar_clinical_significance', getVal: variant => (variant.clinvar || {}).pathogenicity }, { header: 'clinvar_gold_stars', getVal: variant => (variant.clinvar || {}).goldStars }, { header: 'project' }, { header: 'family' }, diff --git a/vlm/clickhouse_utils.py b/vlm/clickhouse_utils.py index dd32573776..75dbb717fe 100644 --- a/vlm/clickhouse_utils.py +++ b/vlm/clickhouse_utils.py @@ -1,5 +1,4 @@ import clickhouse_connect -import hail as hl import os CLICKHOUSE_CONNECTION_PARAMS = { @@ -11,13 +10,12 @@ } -def get_clickhouse_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, genome_build: str) -> hl.Struct: - locus = hl.eval(locus) +def get_clickhouse_variant_counts(chrom: str, pos: int, genome_build: str, ref: str, alt: str) -> tuple[int, int]: client = clickhouse_connect.get_client(**CLICKHOUSE_CONNECTION_PARAMS) results = client.query( "SELECT plus(gt_stats.1, gt_stats.2), plus(gt_stats.3, gt_stats.4) FROM (SELECT dictGet(%(dict_name)s, ('ac_wes', 'ac_wgs', 'hom_wes', 'hom_wgs'), key) AS gt_stats FROM %(table_name)s WHERE variantId=%(variant_id)s)", parameters={ - 'variant_id': f'{locus.contig.replace("chr", "")}-{locus.position}-{ref}-{alt}', + 'variant_id': f'{chrom}-{pos}-{ref}-{alt}', 'table_name': f'{genome_build}/SNV_INDEL/key_lookup', 'dict_name': f'{genome_build}/SNV_INDEL/gt_stats_dict', }, diff --git a/vlm/deploy/Dockerfile b/vlm/deploy/Dockerfile index bda44c60eb..33a0fce9e5 100644 --- a/vlm/deploy/Dockerfile +++ b/vlm/deploy/Dockerfile @@ -1,8 +1,8 @@ -FROM hailgenetics/hail:0.2.128 +FROM python:3.11-slim-bullseye LABEL maintainer="Broad TGG" -RUN pip install --no-cache-dir clickhouse-connect==0.8.18 +RUN pip install --no-cache-dir -r vlm/requirements.txt WORKDIR /vlm diff --git a/vlm/match.py b/vlm/match.py index 07cbe57e60..35f966374f 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -1,14 +1,15 @@ from aiohttp.web import HTTPBadRequest -import hail as hl import os +from pyliftover.liftover import LiftOver import re +from typing import Optional, Tuple from vlm.clickhouse_utils import get_clickhouse_variant_counts SEQR_BASE_URL = os.environ.get('SEQR_BASE_URL') VLM_DEFAULT_CONTACT_EMAIL = os.environ.get('VLM_DEFAULT_CONTACT_EMAIL') NODE_ID = os.environ.get('NODE_ID') - +LIFTOVER_DIR = f'{os.path.dirname(os.path.abspath(__file__))}/liftover_references' BEACON_HANDOVER_TYPE = { 'id': NODE_ID, @@ -37,31 +38,33 @@ 'hg19': GENOME_VERSION_GRCh37, } +CHROMOSOMES = { + '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', + '20', '21', '22', 'X', 'Y', 'M', +} +MIN_POS = 1 +MAX_POS = 300_000_000 + + def get_variant_match(query: dict) -> dict: chrom, pos, ref, alt, genome_build = _parse_match_query(query) - locus = hl.locus(chrom, pos, reference_genome=genome_build) - - ac, hom = get_clickhouse_variant_counts(locus, ref, alt, genome_build) - liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 - liftover_locus = hl.liftover(locus, liftover_genome_build) - lift_ac, lift_hom = get_clickhouse_variant_counts(liftover_locus, ref, alt, liftover_genome_build) + ac, hom = get_clickhouse_variant_counts(chrom, pos, genome_build, ref, alt) + liftover = _liftover_variant(chrom, pos, genome_build) + lift_ac, lift_hom = get_clickhouse_variant_counts(*liftover, ref, alt) if liftover else (0, 0) url = _get_contact_url( - chrom, pos, ref, alt, genome_build, liftover_genome_build, liftover_locus if lift_ac and not ac else None, + chrom, pos, ref, alt, genome_build, liftover if lift_ac and not ac else None, ) return _format_results(ac+lift_ac, hom+lift_hom, url) -def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str, liftover_genome_build: str, liftover_locus: hl.LocusExpression) -> str: +def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str, liftover: Optional[Tuple[str, int, str]]) -> str: if not SEQR_BASE_URL: return SEQR_BASE_URL - if liftover_locus is not None: - lifted = hl.eval(liftover_locus) - chrom = lifted.contig - pos = lifted.position - genome_build = liftover_genome_build + if liftover is not None: + chrom, pos, genome_build = liftover genome_build = genome_build.replace('GRCh', '') return f'{SEQR_BASE_URL}variant_lookup?genomeVersion={genome_build}&variantId={chrom}-{pos}-{ref}-{alt}' @@ -76,16 +79,14 @@ def _parse_match_query(query: dict) -> tuple[str, int, str, str, str]: raise HTTPBadRequest(reason=f'Invalid assemblyId: {query["assemblyId"]}') chrom = query['referenceName'].replace('chr', '') - if genome_build == GENOME_VERSION_GRCh38: - chrom = f'chr{chrom}' - if not hl.eval(hl.is_valid_contig(chrom, reference_genome=genome_build)): + if chrom not in CHROMOSOMES: raise HTTPBadRequest(reason=f'Invalid referenceName: {query["referenceName"]}') start = query['start'] if not start.isnumeric(): raise HTTPBadRequest(reason=f'Invalid start: {start}') start = int(start) - if not hl.eval(hl.is_valid_locus(chrom, start, reference_genome=genome_build)): + if start < MIN_POS or start > MAX_POS: raise HTTPBadRequest(reason=f'Invalid start: {start}') for allele_field in ['referenceBases', 'alternateBases']: @@ -129,3 +130,19 @@ def _format_results(ac: int, hom: int, url: str) -> dict: ] } } + + +LIFTOVERS = { + GENOME_VERSION_GRCh38: None, + GENOME_VERSION_GRCh37: None, +} + + +def _liftover_variant(chrom: str, pos: int, genome_build: str) -> Optional[Tuple[str, int, str]]: + liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 + if not LIFTOVERS[genome_build]: + LIFTOVERS[genome_build] = LiftOver( + f'{LIFTOVER_DIR}/{genome_build.lower()}_to_{liftover_genome_build.lower()}.over.chain.gz' + ) + lifted_coord = LIFTOVERS[genome_build].convert_coordinate(f'chr{chrom}', pos) + return (lifted_coord[0][0].replace('chr', ''), lifted_coord[0][1], liftover_genome_build) if lifted_coord and lifted_coord[0] else None diff --git a/vlm/requirements-test.in b/vlm/requirements-test.in index c63f835ebe..e444f7d6f2 100644 --- a/vlm/requirements-test.in +++ b/vlm/requirements-test.in @@ -1,4 +1,4 @@ +-c vlm/requirements.txt aioresponses coverage<5.2 pytest-aiohttp -clickhouse-connect==0.8.18 diff --git a/vlm/requirements-test.txt b/vlm/requirements-test.txt index ccd84ac4f7..0ae9f4ebc9 100644 --- a/vlm/requirements-test.txt +++ b/vlm/requirements-test.txt @@ -1,58 +1,73 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile vlm/requirements-test.in # -aiohappyeyeballs==2.3.5 - # via aiohttp -aiohttp==3.10.11 +aiohappyeyeballs==2.6.2 # via + # -c vlm/requirements.txt + # aiohttp +aiohttp==3.13.5 + # via + # -c vlm/requirements.txt # aioresponses # pytest-aiohttp aioresponses==0.7.8 # via -r vlm/requirements-test.in -aiosignal==1.3.1 - # via aiohttp -async-timeout==4.0.2 - # via aiohttp -attrs==23.1.0 - # via aiohttp -clickhouse-connect==0.8.18 - # via -r vlm/requirements-test.in +aiosignal==1.4.0 + # via + # -c vlm/requirements.txt + # aiohttp +attrs==26.1.0 + # via + # -c vlm/requirements.txt + # aiohttp coverage==5.1 # via -r vlm/requirements-test.in -exceptiongroup==1.1.3 - # via pytest -frozenlist==1.3.3 +frozenlist==1.8.0 # via + # -c vlm/requirements.txt # aiohttp # aiosignal -idna==3.7 - # via yarl -iniconfig==2.0.0 +idna==3.15 + # via + # -c vlm/requirements.txt + # yarl +iniconfig==2.3.0 # via pytest -multidict==6.0.4 +multidict==6.7.1 # via + # -c vlm/requirements.txt # aiohttp # yarl -packaging==23.1 +packaging==26.2 # via # aioresponses # pytest -pluggy==1.2.0 +pluggy==1.6.0 # via pytest -propcache==0.2.0 - # via yarl -pytest==7.4.0 +propcache==0.5.2 + # via + # -c vlm/requirements.txt + # aiohttp + # yarl +pygments==2.20.0 + # via pytest +pytest==9.0.3 # via # pytest-aiohttp # pytest-asyncio -pytest-aiohttp==1.0.4 +pytest-aiohttp==1.1.0 # via -r vlm/requirements-test.in -pytest-asyncio==0.21.0 +pytest-asyncio==1.3.0 # via pytest-aiohttp -tomli==2.0.1 - # via pytest -yarl==1.17.2 - # via aiohttp +typing-extensions==4.15.0 + # via + # -c vlm/requirements.txt + # aiosignal + # pytest-asyncio +yarl==1.24.2 + # via + # -c vlm/requirements.txt + # aiohttp diff --git a/vlm/requirements.in b/vlm/requirements.in new file mode 100644 index 0000000000..8f8e52a40f --- /dev/null +++ b/vlm/requirements.in @@ -0,0 +1,4 @@ +aiohttp +clickhouse-connect==0.8.18 +pyjwt +pyliftover diff --git a/vlm/requirements.txt b/vlm/requirements.txt new file mode 100644 index 0000000000..d563dd3030 --- /dev/null +++ b/vlm/requirements.txt @@ -0,0 +1,48 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile vlm/requirements.in +# +aiohappyeyeballs==2.6.2 + # via aiohttp +aiohttp==3.13.5 + # via -r vlm/requirements.in +aiosignal==1.4.0 + # via aiohttp +attrs==26.1.0 + # via aiohttp +certifi==2026.5.20 + # via clickhouse-connect +clickhouse-connect==0.8.18 + # via -r vlm/requirements.in +frozenlist==1.8.0 + # via + # aiohttp + # aiosignal +idna==3.15 + # via yarl +lz4==4.4.5 + # via clickhouse-connect +multidict==6.7.1 + # via + # aiohttp + # yarl +propcache==0.5.2 + # via + # aiohttp + # yarl +pyjwt==2.12.1 + # via -r vlm/requirements.in +pyliftover==0.4.1 + # via -r vlm/requirements.in +pytz==2026.2 + # via clickhouse-connect +typing-extensions==4.15.0 + # via aiosignal +urllib3==2.7.0 + # via clickhouse-connect +yarl==1.24.2 + # via aiohttp +zstandard==0.25.0 + # via clickhouse-connect diff --git a/vlm/test_vlm.py b/vlm/test_vlm.py index 9b0a1f02e3..02134c5e70 100644 --- a/vlm/test_vlm.py +++ b/vlm/test_vlm.py @@ -52,7 +52,7 @@ async def test_match(self, mocked_responses): 'id': 'TestVLM', 'label': 'TestVLM browser', }, - 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=38&variantId=chr1-38724419-T-G', + 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=38&variantId=1-38724419-T-G', 'email': None, } ], @@ -195,7 +195,7 @@ async def test_match(self, mocked_responses): 'id': 'TestVLM', 'label': 'TestVLM browser', }, - 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=38&variantId=chr7-143270172-A-G', + 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=38&variantId=7-143270172-A-G', 'email': None, } ], diff --git a/vlm/web_app.py b/vlm/web_app.py index 72875e5817..90a5594f0c 100644 --- a/vlm/web_app.py +++ b/vlm/web_app.py @@ -1,20 +1,12 @@ from aiohttp import web -import hail as hl import logging -import os import traceback from vlm.auth import authenticate -from vlm.match import get_variant_match, GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 +from vlm.match import get_variant_match logger = logging.getLogger(__name__) -JAVA_OPTS_XSS = os.environ.get('JAVA_OPTS_XSS') -MACHINE_MEM = os.environ.get('MACHINE_MEM') -JVM_MEMORY_FRACTION = 0.9 - -LIFTOVER_DIR = f'{os.path.dirname(os.path.abspath(__file__))}/liftover_references' - def _handle_exception(e, request): logger.error(f'{request.headers.get("From")} "{e}"') @@ -42,22 +34,6 @@ async def match(request: web.Request) -> web.Response: async def init_web_app(): - spark_conf = {} - # memory limits adapted from https://github.com/hail-is/hail/blob/main/hail/python/hailtop/hailctl/dataproc/start.py#L321C17-L321C36 - if MACHINE_MEM: - spark_conf['spark.driver.memory'] = f'{int((int(MACHINE_MEM) - 11) * JVM_MEMORY_FRACTION)}g' - if JAVA_OPTS_XSS: - spark_conf.update( - {f'spark.{field}.extraJavaOptions': f'-Xss{JAVA_OPTS_XSS}' for field in ['driver', 'executor']}) - hl.init(idempotent=True, spark_conf=spark_conf or None) - - rg37 = hl.get_reference(GENOME_VERSION_GRCh37) - rg38 = hl.get_reference(GENOME_VERSION_GRCh38) - if not rg37.has_liftover(rg38): - rg37.add_liftover(f'{LIFTOVER_DIR}/grch37_to_grch38.over.chain.gz', rg38) - if not rg38.has_liftover(rg37): - rg38.add_liftover(f'{LIFTOVER_DIR}/grch38_to_grch37.over.chain.gz', rg37) - app = web.Application(middlewares=[error_middleware], client_max_size=(1024 ** 2) * 10) app.add_routes([ web.get('/vlm/match', match),