diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index c9293b7924..7924553107 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -1546,7 +1546,7 @@ def _prefilter_entries(self, entries, freqs=None, **kwargs): def _filter_frequency(self, entries, frequencies, pathogenicity=None, sample_data=None, **kwargs): callset_filter = frequencies.get(self.callset_filter_field) or {} - if (not callset_filter.get('ac') or callset_filter['ac'] >= self.MIN_MULTI_FAMILY_SEQR_AC) and ( + if (not callset_filter.get('ac') or callset_filter['ac'] > self.MIN_MULTI_FAMILY_SEQR_AC) and ( len(sample_data['sample_type_families']) > 1 or len(next(iter(sample_data['sample_type_families'].values()))) > 1 ): raise InvalidSearchException( diff --git a/clickhouse_search/migrations/0040_gnomadnoncodingconstraintdict.py b/clickhouse_search/migrations/0040_gnomadnoncodingconstraintdict.py index 3c8e29e036..35376dfd70 100644 --- a/clickhouse_search/migrations/0040_gnomadnoncodingconstraintdict.py +++ b/clickhouse_search/migrations/0040_gnomadnoncodingconstraintdict.py @@ -418,7 +418,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/SNV_INDEL/entries', - 'engine': clickhouse_search.backend.engines.CollapsingMergeTree('sign', deduplicate_merge_projection_mode='rebuild', index_granularity=8192, order_by=('project_guid', 'family_guid', 'sample_type', 'is_gnomad_gt_5_percent', 'is_annotated_in_any_gene', 'key'), partition_by='project_guid'), + 'engine': clickhouse_search.backend.engines.CollapsingMergeTree('sign', deduplicate_merge_projection_mode='rebuild', index_granularity=8192, order_by=('project_guid', 'family_guid', 'sample_type', 'is_gnomad_gt_5_percent', 'is_annotated_in_any_gene', 'key'), partition_by=['project_guid', 'partition_id']), 'projection': clickhouse_search.models.search_models.Projection('xpos_projection', order_by='is_gnomad_gt_5_percent, is_annotated_in_any_gene, xpos'), }, managers=[ diff --git a/clickhouse_search/models/search_models.py b/clickhouse_search/models/search_models.py index bcd47b2dcc..5891057573 100644 --- a/clickhouse_search/models/search_models.py +++ b/clickhouse_search/models/search_models.py @@ -359,7 +359,7 @@ class Meta: engine = CollapsingMergeTree( 'sign', order_by=('project_guid', 'family_guid', 'sample_type', 'is_gnomad_gt_5_percent', 'is_annotated_in_any_gene', 'key'), - partition_by='project_guid, partition_id', + partition_by=['project_guid', 'partition_id'], deduplicate_merge_projection_mode='rebuild', index_granularity=8192, ) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 71606ab2de..d3368c7457 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -5,6 +5,7 @@ from django.db import connections from django.db.models import Count, F, Min, Q from django.db.models.functions import JSONObject +from django.db.utils import OperationalError import json from clickhouse_search.backend.fields import NamedTupleField @@ -13,9 +14,9 @@ from clickhouse_search.managers import InvalidDatasetTypeException, InvalidSearchException from clickhouse_search.models.gt_stats_models import PROJECT_GT_STATS_VIEW_CLASS_MAP from clickhouse_search.models.reference_data_models import BaseClinvar, BaseHgmd -from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, \ +from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, EntriesSnvIndel, \ ENTRY_CLASS_MAP, VARIANTS_CLASS_MAP, VARIANT_DETAILS_CLASS_MAP -from reference_data.models import GeneInfo, GeneConstraint, Omim, GENOME_VERSION_LOOKUP +from reference_data.models import GeneInfo, GeneConstraint, Omim, GENOME_VERSION_LOOKUP, GENOME_VERSION_GRCh38 from seqr.models import Sample, PhenotypePrioritization, Family, Individual from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.logging_utils import SeqrLogger @@ -1030,8 +1031,20 @@ def delete_clickhouse_project(project, dataset_type, sample_type=None): if dataset_type == Sample.DATASET_TYPE_SV_CALLS and sample_type == Sample.SAMPLE_TYPE_WES: dataset_type = 'GCNV' table_base = f'{GENOME_VERSION_LOOKUP[project.genome_version]}/{dataset_type}' + partition_id = f"'{project.guid}'" + partition_ids = [partition_id] + if project.genome_version == GENOME_VERSION_GRCh38 and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: + partition_ids = [ + f'({partition_id}, {pid})' for pid in + EntriesSnvIndel.objects.filter(project_guid=project.guid).values_list('partition_id', flat=True).distinct() + ] with connections['clickhouse_write'].cursor() as cursor: - cursor.execute(f'ALTER TABLE "{table_base}/entries" DROP PARTITION %s', [project.guid]) + for partition_id in partition_ids: + try: + cursor.execute(f"ALTER TABLE `{table_base}/entries` DROP PARTITION {partition_id}") + except OperationalError: + # Repartitioning of this table was not rolled out via a standard migration and may be skipped in some local installs + raise OperationalError('Unable to delete project data due to a partitioning migration error. Please reach out to the seqr team for assistance.') if dataset_type != 'GCNV': cursor.execute(f'ALTER TABLE "{table_base}/project_gt_stats" DROP PARTITION %s', [project.guid]) PROJECT_GT_STATS_VIEW_CLASS_MAP[project.genome_version][dataset_type].refresh() diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 6b8188e078..6307fce27e 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -168,7 +168,8 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun response, search_hash, search_body = self._execute_search(project_families=project_families, sort=sort, **kwargs) self.assertEqual(response.status_code, 200) expected_response = { - 'searchedVariants': results_page or expected_results, + 'searchedVariantIds': [], + 'variantsById': {}, 'search': { 'search': {**search_body, **(response_search or {})}, 'projectFamilies': DEFAULT_PROJECT_FAMILIES if project_families is None else project_families, @@ -176,6 +177,13 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun }, } + for variant in results_page or expected_results: + if isinstance(variant, list): + expected_response['searchedVariantIds'].append([v['variantId'] for v in variant]) + expected_response['variantsById'].update({v['variantId']: v for v in variant}) + else: + expected_response['searchedVariantIds'].append(variant['variantId']) + expected_response['variantsById'][variant['variantId']] = variant if expected_results: expected_response.update({ **{key: mock.ANY for key in SEARCH_RESPONSE_KEYS}, @@ -214,7 +222,7 @@ def _assert_expected_export_data(self, search_hash, export_data): url = reverse(export_variants_handler, args=[search_hash]) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertListEqual([line.split('\t') for line in response.content.decode().strip().split('\n')], export_data) + self.assertListEqual([line.split('\t') for line in response.content.decode().strip('\n').split('\n')], export_data) def _assert_expected_search_error(self, error, **kwargs): response, search_hash, _ = self._execute_search(**kwargs) @@ -359,8 +367,12 @@ def test_multi_project_search(self): ) def test_all_project_search(self): - self.login_collaborator() request_body = {'allGenomeProjectFamilies': '38'} + self._assert_expected_search_error( + 'No data available for genome version "GRCh38"', request_body=request_body, check_login=self.check_require_login, + ) + + self.login_collaborator() project_families = [{'projectGuid': 'R0001_1kg', 'familyGuids': [ 'F000001_1', 'F000002_2', 'F000003_3', 'F000004_4', 'F000005_5', 'F000006_6', 'F000007_7', 'F000008_8', 'F000009_9', 'F000010_10', 'F000013_13', @@ -1203,11 +1215,11 @@ def test_variant_lookup(self, mock_liftover): }]) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json()['variants'][0], { + self.assertDictEqual(response.json()['variantsById'], {'1-10439-AC-A': { **VARIANT1, 'familyGuids': [], 'lookupFamilyGuids': VARIANT1['familyGuids'], - }) + }}) cache_key = 'variant_lookup_results__7-143270172-A-G__37' self._assert_expected_lookup( @@ -1326,11 +1338,11 @@ def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version 'individualsByGuid': expected_individuals or {i: mock.ANY for i in individual_guids or []}, 'locusListsByGuid': {'LL00049_pid_genes_autosomal_do': mock.ANY, 'LL00005_retina_proteome': mock.ANY}, 'totalSampleCounts': {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3, 'WGS': 3}} if genome_version == '38' else {}, - 'variants': [{ + 'variantsById': {v['variantId']: { **v, 'familyGuids': [], 'lookupFamilyGuids': v['familyGuids'] + v.get('liftedFamilyGuids', []), - } for v in variants], + } for v in variants}, **kwargs, } self.assertDictEqual(response.json(), expected_body) @@ -1362,7 +1374,8 @@ def test_get_single_variant(self): response = self.client.get(url) self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), { - 'searchedVariants': [FAMILY_1_VARIANT], + 'searchedVariantIds': ['21-3343353-GAGA-G'], + 'variantsById': {'21-3343353-GAGA-G': FAMILY_1_VARIANT}, 'familiesByGuid': {'F000001_1': mock.ANY}, 'projectsByGuid': {'R0001_1kg': mock.ANY}, 'locusListsByGuid': {'LL00049_pid_genes_autosomal_do': { @@ -1379,21 +1392,21 @@ def test_get_single_variant(self): response = self.client.get(url_template.format('M-4429-G-A', 'F000002_2')) self.assertEqual(response.status_code, 200) - self.assertListEqual(response.json()['searchedVariants'], [MITO_VARIANT1]) + self.assertDictEqual(response.json()['variantsById'], {'M-4429-G-A': MITO_VARIANT1}) response = self.client.get(url_template.format('suffix_140608_DUP', 'F000002_2')) self.assertEqual(response.status_code, 200) - self.assertListEqual(response.json()['searchedVariants'], [GCNV_VARIANT4]) + self.assertDictEqual(response.json()['variantsById'], {'suffix_140608_DUP': GCNV_VARIANT4}) self.login_manager() response = self.client.get(url_template.format('phase2_DEL_chr14_4640', 'F000014_14')) self.assertEqual(response.status_code, 200) - self.assertListEqual(response.json()['searchedVariants'], [SV_VARIANT4]) + self.assertDictEqual(response.json()['variantsById'], {'phase2_DEL_chr14_4640': SV_VARIANT4}) self._set_grch37_search() response = self.client.get(url_template.format('7-143270172-A-G', 'F000002_2')) self.assertEqual(response.status_code, 200) - self.assertListEqual(response.json()['searchedVariants'], [GRCH37_VARIANT]) + self.assertDictEqual(response.json()['variantsById'], {'7-143270172-A-G': GRCH37_VARIANT}) self.mock_redis.get.assert_not_called() self.mock_redis.set.assert_not_called() @@ -2137,7 +2150,9 @@ def test_gene_variant_lookup(self): self._assert_expected_search( [variant4], request_body=request_body, response_search=response_search, cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}], - annotations=annotations, freqs=freqs, locus=locus, project_families=[], + annotations=annotations, freqs=freqs, locus=locus, project_families=[], export_data=[ + EXPORT_DATA[0][:27], EXPORT_DATA[4][:24] + ['3 Families', '', ''], + ], ) freqs = {'callset': freqs['callset']} @@ -2357,16 +2372,16 @@ def test_search_context(self): }]) response, _, _ = self._execute_search(search_hash=9876) self.assertEqual(response.status_code, 200) - variants = response.json()['searchedVariants'] + variants = response.json()['variantsById'] self.assertEqual(len(variants), 1) - self.assertFalse('discoveryTags' in variants[0]) + self.assertFalse('discoveryTags' in variants['1-248367227-TC-T']) self.login_analyst_user() response, _, _ = self._execute_search(search_hash=9876) self.assertEqual(response.status_code, 200) - variants = response.json()['searchedVariants'] + variants = response.json()['variantsById'] self.assertEqual(len(variants), 1) - self.assertListEqual(variants[0]['discoveryTags'], [{ + self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], [{ 'savedVariant': { 'variantGuid': 'SV0000006_1248367227_r0003_tes', 'familyGuid': 'F000012_12', diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 91f9ced017..b43764cd73 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1464,7 +1464,7 @@ def format_cached_variant(variant): '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], ['M', '14783', 'T', 'C', 'ENSG00000198727', '', 'synonymous_variant', '1', '', '', '', '', '', '', '', '', '', '', '', 'rs193302982', 'ENST00000361789.2:c.37T>C', 'ENSP00000354554.2:p.Leu13=', 'Likely_pathogenic', '0', '2', '', '', '', - '', '', 'HG00731', '2', 'artifact_prone_site'], + '', '', 'HG00731', '2', 'artifact_prone_site', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''], ] SPLIT_FAMILY_EXPORT_DATA = [ diff --git a/matchmaker/views/matchmaker_api.py b/matchmaker/views/matchmaker_api.py index a1791115fd..7c7abadc07 100644 --- a/matchmaker/views/matchmaker_api.py +++ b/matchmaker/views/matchmaker_api.py @@ -2,7 +2,7 @@ import requests from datetime import datetime from django.core.mail.message import EmailMessage -from django.db.models import prefetch_related_objects, Q, Value, CharField +from django.db.models import prefetch_related_objects, F, Q, Value, CharField from django.db.models.functions import Coalesce from matchmaker.models import MatchmakerResult, MatchmakerContactNotes, MatchmakerSubmission, MatchmakerSubmissionGenes, \ @@ -58,14 +58,16 @@ def get_individual_mme_matches(request, submission_guid): variant_guids = {gv['variantGuid'] for gv in gene_variants} variants = get_json_for_saved_variants( - SavedVariant.objects.filter(guid__in=variant_guids), genome_version=project.genome_version, additional_values={ + SavedVariant.objects.filter(guid__in=variant_guids), additional_values={ 'genomeVersion': Coalesce('saved_variant_json__genomeVersion', Value(project.genome_version), output_field=CharField()), + 'selectedMainTranscript': F('main_transcript'), + 'xposEnd': F('xpos_end'), }, ) response_json['savedVariantsByGuid'] = {} for variant in variants: - chrom, pos = get_chrom_pos(variant['xpos']) - variant.update({'chrom': chrom, 'pos': pos}) + _, end = get_chrom_pos(variant.pop('xposEnd')) + variant.update({'end': end}) response_json['savedVariantsByGuid'][variant['variantGuid']] = variant return _parse_mme_results( diff --git a/matchmaker/views/matchmaker_api_tests.py b/matchmaker/views/matchmaker_api_tests.py index 0bbe9e53d7..267c765fb8 100644 --- a/matchmaker/views/matchmaker_api_tests.py +++ b/matchmaker/views/matchmaker_api_tests.py @@ -231,7 +231,7 @@ def test_get_individual_mme_matches(self): set(response_json['savedVariantsByGuid'].keys()), {'SV0000001_2103343353_r0390_100'} ) - saved_variant_fields = {*SAVED_VARIANT_FIELDS, 'chrom', 'pos', 'genomeVersion'} + saved_variant_fields = {*SAVED_VARIANT_FIELDS, 'selectedMainTranscript', 'end', 'genomeVersion'} self.assertSetEqual(set(response_json['savedVariantsByGuid']['SV0000001_2103343353_r0390_100'].keys()), saved_variant_fields) self.assertDictEqual(response_json['mmeContactNotes'], {}) diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index 9e11b9296a..a3150444c7 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -240,6 +240,7 @@ 'family_filter': { CONFIRMED_FAMILY_FILTER: True }, + 'inheritance': {'mode': COMPOUND_HET}, 'annotations': { 'splice_ai': 0.5, }, diff --git a/seqr/models.py b/seqr/models.py index e7ab32f379..4b625edc7a 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -862,7 +862,8 @@ def __unicode__(self): class Meta: unique_together = ('xpos', 'xpos_end', 'variant_id', 'family') - json_fields = ['guid', 'xpos', 'ref', 'alt', 'variant_id', 'selected_main_transcript_id', 'acmg_classification'] + json_fields = ['guid', 'key', 'xpos', 'ref', 'alt', 'variant_id', 'selected_main_transcript_id', 'acmg_classification', + 'sv_type', 'gene_ids', 'genotypes'] class VariantTagType(ModelWithGUID): diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index 7f1dc425f3..165cd931bc 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -943,10 +943,14 @@ def _add_row(row, family_id, row_type): for data_type, analysed in analysed_by_family_type[family_id].items() ] inheritance_models = f.pop('inheritance_models', []) + family_structure = _get_family_structure( + sum(1 for individual in individuals_by_id.values() if individual.get('date_data_generation')), + sum(1 for id in known_ids.values() if individuals_by_id.get(id, {}).get('date_data_generation')), + ) f.update({ 'individual_count': len(individuals_by_id), 'other_individual_ids': '; '.join(sorted(individuals_ids)), - 'family_structure': _get_family_structure(len(individuals_by_id), sum(1 for id in known_ids.values() if id)), + 'family_structure': family_structure, 'data_type': earliest_sample.get('data_type'), 'date_data_generation': earliest_sample.get('date_data_generation'), 'genes': '; '.join(sorted(f.get('genes', []))), diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index e52abc78c1..729b3ad93d 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -1294,6 +1294,28 @@ def test_family_metadata(self): 'known_condition_name': '', 'condition_inheritance': 'Unknown', }) + test_no_data_row = next(r for r in response_json['rows'] if r['familyGuid'] == 'F000009_9') + self.assertDictEqual(test_no_data_row, { + 'projectGuid': 'R0001_1kg', + 'internal_project_id': '1kg project nåme with uniçøde', + 'familyGuid': 'F000009_9', + 'family_id': '9', + 'displayName': '9', + 'solve_status': 'Unsolved', + 'actual_inheritance': '', + 'date_data_generation': None, + 'data_type': None, + 'other_individual_ids': 'NA20878', + 'individual_count': 1, + 'family_structure': 'other', + 'genes': '', + 'pmid_id': None, + 'phenotype_description': None, + 'analysisStatus': 'Q', + 'analysis_groups': '', + 'analysed_by': '', + 'consanguinity': 'Unknown', + }) # Test empty project empty_project_url = reverse(family_metadata, args=['R0002_empty']) diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 3a1b289226..6daa725fc7 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -93,7 +93,7 @@ def create_manual_saved_variant_handler(request, family_guid): saved_variants = SavedVariant.objects.filter(family=family, guid__in=saved_variant_guids) _update_tags(saved_variants, {'tags': tags}, request.user) - response = get_json_for_saved_variants_with_tags(saved_variants, add_details=True, genome_version=genome_version) + response = get_json_for_saved_variants_with_tags(saved_variants) return create_json_response(response) @@ -125,7 +125,7 @@ def create_saved_variant_handler(request): elif variant_json.get('tags'): _update_tags(saved_variants, variant_json, request.user) - response.update(get_json_for_saved_variants_with_tags(saved_variants, add_details=True, genome_version=family.project.genome_version)) + response.update(get_json_for_saved_variants_with_tags(saved_variants)) return create_json_response(response) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 80549e3dc1..7382a100a2 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -1,4 +1,3 @@ -from copy import deepcopy import json import mock @@ -12,7 +11,7 @@ from seqr.views.utils.orm_to_json_utils import get_json_for_saved_variants from seqr.views.utils.test_utils import AuthenticationTestCase, SAVED_VARIANT_DETAIL_FIELDS, TAG_FIELDS, GENE_VARIANT_FIELDS, \ TAG_TYPE_FIELDS, LOCUS_LIST_FIELDS, PA_LOCUS_LIST_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \ - FAMILY_NOTE_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, AnvilAuthenticationTestCase + FAMILY_NOTE_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, SAVED_VARIANT_FIELDS, AnvilAuthenticationTestCase PROJECT_GUID = 'R0001_1kg' @@ -27,7 +26,7 @@ GENE_GUID_2 = 'ENSG00000197530' VARIANT_TAG_RESPONSE_KEYS = { - 'variantTagsByGuid', 'variantNotesByGuid', 'variantFunctionalDataByGuid', 'savedVariantsByGuid', + 'variantTagsByGuid', 'variantNotesByGuid', 'variantFunctionalDataByGuid', 'savedVariantsByGuid', 'variantsById', } SAVED_VARIANT_RESPONSE_KEYS = { *VARIANT_TAG_RESPONSE_KEYS, 'familiesByGuid', 'omimIntervals', 'totalSampleCounts', @@ -158,12 +157,12 @@ def test_saved_variant_data(self): response_json = response.json() self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS) - variants = response_json['savedVariantsByGuid'] - self.assertSetEqual(set(variants.keys()), {'SV0000002_1248367227_r0390_100', VARIANT_GUID}) + saved_variants = response_json['savedVariantsByGuid'] + self.assertSetEqual(set(saved_variants.keys()), {'SV0000002_1248367227_r0390_100', VARIANT_GUID}) - variant = variants[VARIANT_GUID] - fields = {*SAVED_VARIANT_DETAIL_FIELDS, 'key', 'mainTranscriptId'} - self.assertSetEqual(set(variants['SV0000002_1248367227_r0390_100'].keys()), fields) + variant = saved_variants[VARIANT_GUID] + fields = {*SAVED_VARIANT_FIELDS, 'tagGuids', 'functionalDataGuids', 'noteGuids'} + self.assertSetEqual(set(saved_variants['SV0000002_1248367227_r0390_100'].keys()), fields) self.assertSetEqual(set(variant.keys()), {*fields, 'mmeSubmissions'}) self.assertListEqual(variant['familyGuids'], ['F000001_1']) self.assertSetEqual(set(variant['genotypes'].keys()), {'I000003_na19679', 'I000001_na19675', 'I000002_na19678'}) @@ -175,6 +174,14 @@ def test_saved_variant_data(self): {'geneId': 'ENSG00000135953', 'submissionGuid': 'MS000001_na19675', 'variantGuid': VARIANT_GUID} ]) + variants = response_json['variantsById'] + self.assertSetEqual(set(variants.keys()), {'1-248367227-TC-T', '21-3343353-GAGA-G'}) + variant = variants['21-3343353-GAGA-G'] + variant_fields = {*SAVED_VARIANT_DETAIL_FIELDS, 'mainTranscriptId'} + self.assertSetEqual(set(variant.keys()), variant_fields) + self.assertListEqual(variant['familyGuids'], ['F000001_1']) + self.assertSetEqual(set(variant['genotypes'].keys()), {'I000003_na19679', 'I000001_na19675', 'I000002_na19678'}) + tag = response_json['variantTagsByGuid']['VT1708633_2103343353_r0390_100'] self.assertSetEqual(set(tag.keys()), TAG_FIELDS) @@ -317,11 +324,12 @@ def test_saved_variant_data(self): self.assertEqual(response.status_code, 200) response_json = response.json() self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS) - variants = response_json['savedVariantsByGuid'] self.assertSetEqual( - set(variants.keys()), + set(response_json['savedVariantsByGuid'].keys()), {'SV0000002_1248367227_r0390_100', VARIANT_GUID} ) + variants = response_json['variantsById'] + self.assertSetEqual(set(variants.keys()), {'1-248367227-TC-T', '21-3343353-GAGA-G'}) discovery_tags = [{ 'savedVariant': { 'variantGuid': 'SV0000006_1248367227_r0003_tes', @@ -337,8 +345,8 @@ def test_saved_variant_data(self): 'lastModifiedDate': '2018-05-29T16:32:51.449Z', 'createdBy': None, }] - self.assertListEqual(variants['SV0000002_1248367227_r0390_100']['discoveryTags'], discovery_tags) - self.assertListEqual(variants['SV0000002_1248367227_r0390_100']['familyGuids'], ['F000002_2']) + self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], discovery_tags) + self.assertListEqual(variants['1-248367227-TC-T']['familyGuids'], ['F000002_2']) self.assertSetEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1', 'F000012_12'}) self.assertSetEqual(set(response_json['familiesByGuid']['F000012_12'].keys()), FAMILY_FIELDS) self.assertDictEqual(response_json['familiesByGuid']['F000001_1'], {'tpmGenes': ['ENSG00000135953']}) @@ -348,12 +356,10 @@ def test_saved_variant_data(self): self.assertEqual(response.status_code, 200) response_json = response.json() self.assertSetEqual(set(response_json.keys()), family_context_response_keys) - self.assertSetEqual( - set(response_json['savedVariantsByGuid'].keys()), - {'SV0000002_1248367227_r0390_100', 'SV0000001_2103343353_r0390_100'} - ) - self.assertListEqual(variants['SV0000002_1248367227_r0390_100']['discoveryTags'], discovery_tags) - self.assertListEqual(variants['SV0000002_1248367227_r0390_100']['familyGuids'], ['F000002_2']) + variants = response_json['variantsById'] + self.assertSetEqual(set(variants.keys()), {'1-248367227-TC-T', '21-3343353-GAGA-G'}) + self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], discovery_tags) + self.assertListEqual(variants['1-248367227-TC-T']['familyGuids'], ['F000002_2']) self.assertEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1', 'F000002_2', 'F000012_12'}) # Test empty project @@ -373,8 +379,9 @@ def test_saved_variant_data(self): self.assertEqual(response.status_code, 200) response_json = response.json() self.assertSetEqual(set(response_json.keys()), SAVED_VARIANT_RESPONSE_KEYS - {'omimIntervals'}) - self.assertSetEqual(set(response_json['savedVariantsByGuid']['SV0000002_1248367227_r0390_100'].keys()), { - *fields, 'discoveryTags', 'screenRegionType', 'sortedRegulatoryFeatureConsequences', 'sortedMotifFeatureConsequences', + self.assertSetEqual(set(response_json['savedVariantsByGuid']['SV0000002_1248367227_r0390_100'].keys()), fields) + self.assertSetEqual(set(response_json['variantsById']['1-248367227-TC-T'].keys()), { + *variant_fields, 'discoveryTags', 'screenRegionType', 'sortedRegulatoryFeatureConsequences', 'sortedMotifFeatureConsequences', }) def test_create_saved_variant(self): @@ -402,15 +409,15 @@ def test_create_saved_variant(self): ) variant_json = { - **CREATE_VARIANT_JSON, + **{k: v for k, v in CREATE_VARIANT_JSON.items() if k in SAVED_VARIANT_FIELDS}, 'variantGuid': variant_guid, 'familyGuids': ['F000001_1'], 'acmgClassification': None, 'selectedMainTranscriptId': None, 'noteGuids': [], 'functionalDataGuids': [], - 'transcripts': mock.ANY, - 'mainTranscriptId': mock.ANY, + 'geneIds': ['ENSG00000277258', 'ENSG00000177000'], + 'svType': None, } response_json = response.json() response_variant_json = response_json['savedVariantsByGuid'][variant_guid] @@ -481,7 +488,9 @@ def test_create_saved_sv_variant(self): self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), {*VARIANT_TAG_RESPONSE_KEYS, 'genesById'}) + response_keys = {*VARIANT_TAG_RESPONSE_KEYS, 'genesById'} + response_keys.remove('variantsById') + self.assertSetEqual(set(response_json.keys()), response_keys) self.assertEqual(len(response_json['savedVariantsByGuid']), 1) variant_guid = next(iter(response_json['savedVariantsByGuid'])) @@ -489,7 +498,8 @@ def test_create_saved_sv_variant(self): self._assert_created_variant(saved_variant, variant_json, dataset_type='SV_WES', sv_type='DUP', gene_ids=['ENSG00000240361']) self.assertEqual(saved_variant.xpos_end, 2061414175) - variant_json.update({ + expected_variant_json = { + **{k: v for k, v in variant_json.items() if k in SAVED_VARIANT_FIELDS}, 'variantGuid': variant_guid, 'familyGuids': ['F000001_1'], 'alt': None, @@ -497,10 +507,11 @@ def test_create_saved_sv_variant(self): 'selectedMainTranscriptId': None, 'tagGuids': [], 'functionalDataGuids': [], - }) + 'geneIds': ['ENSG00000240361'], + } response_variant_json = response_json['savedVariantsByGuid'][variant_guid] notes = [response_json['variantNotesByGuid'][note_guid] for note_guid in response_variant_json.pop('noteGuids')] - self.assertDictEqual(variant_json, response_variant_json) + self.assertDictEqual(expected_variant_json, response_variant_json) self.assertListEqual(['A promising SV'], [note['note'] for note in notes]) self.assertDictEqual(response_json['genesById'], {'ENSG00000240361': {'notes': [mock.ANY]}}) self.assertEqual(response_json['genesById']['ENSG00000240361']['notes'][0]['note'], 'A promising SV') @@ -537,17 +548,17 @@ def test_create_saved_compound_hets(self): 'selectedMainTranscriptId': None, 'noteGuids': [], 'functionalDataGuids': [], - 'predictions': mock.ANY, - 'populations': mock.ANY, + 'geneIds': [], + 'svType': None, } expected_compound_het_3_json = { 'variantGuid': new_compound_het_3_guid, - **COMPOUND_HET_3_JSON, + **{k: v for k, v in COMPOUND_HET_3_JSON.items() if k in SAVED_VARIANT_FIELDS}, **expected_json, } expected_compound_het_4_json = { 'variantGuid': new_compound_het_4_guid, - **COMPOUND_HET_4_JSON, + **{k: v for k, v in COMPOUND_HET_4_JSON.items() if k in SAVED_VARIANT_FIELDS}, **expected_json, } response_json = response.json() @@ -617,8 +628,9 @@ def test_create_manual_variant(self): 'key': None, 'noteGuids': [], 'functionalDataGuids': [], + 'geneIds': mock.ANY, + 'svType': None, } - variant_json.update(base_variant_json) response_json = response.json() response_variant_json = response_json['savedVariantsByGuid'][variant_guid] tag_guids = response_variant_json.pop('tagGuids') @@ -626,7 +638,7 @@ def test_create_manual_variant(self): tag = response_json['variantTagsByGuid'][tag_guids[0]] self.assertEqual('Review', tag['name']) self.assertListEqual([variant_guid], tag['variantGuids']) - self.assertDictEqual(variant_json, response_variant_json) + self.assertDictEqual({**{k: v for k, v in variant_json.items() if k in SAVED_VARIANT_FIELDS}, **base_variant_json}, response_variant_json) # Create manual SV manual_variant_request_body = { @@ -661,13 +673,15 @@ def test_create_manual_variant(self): } self._assert_created_variant(saved_sv_variant, sv_variant_json, gene_ids=['ENSG00000240361'], dataset_type='SV', sv_type='DEL', has_saved_variant_json=True) - sv_variant_json.update(base_variant_json) - sv_variant_json.update({ + sv_variant_json = { + **{k: v for k, v in sv_variant_json.items() if k in SAVED_VARIANT_FIELDS}, + **base_variant_json, 'variantGuid': sv_variant_guid, 'tagGuids': [mock.ANY], 'ref': None, 'alt': None, - }) + 'svType': 'DEL', + } response_json = response.json() response_variant_json = response_json['savedVariantsByGuid'][sv_variant_guid] self.assertDictEqual(sv_variant_json, response_variant_json) @@ -787,13 +801,13 @@ def test_create_partially_saved_compound_het_variant_note(self): self._assert_created_variant(saved_compound_het_5, COMPOUND_HET_4_JSON) expected_compound_het_5_json = { - **COMPOUND_HET_4_JSON, + **{k: v for k, v in COMPOUND_HET_4_JSON.items() if k in SAVED_VARIANT_FIELDS}, 'variantGuid': compound_het_5_guid, 'selectedMainTranscriptId': None, 'tagGuids': [], 'functionalDataGuids': [], - 'predictions': mock.ANY, - 'populations': mock.ANY, + 'geneIds': [], + 'svType': None, } response_json = response.json() response_compound_het_5_json = response_json['savedVariantsByGuid'][compound_het_5_guid] @@ -1103,7 +1117,7 @@ def test_update_variant_main_transcript(self): self.assertEqual(len(saved_variants), 1) self.assertEqual(saved_variants.first().selected_main_transcript_id, transcript_id) self.assertDictEqual(saved_variants.first().main_transcript, transcript) - self.assertEqual(get_json_for_saved_variants(saved_variants, add_details=True)[0]['selectedMainTranscriptId'], transcript_id) + self.assertEqual(get_json_for_saved_variants(saved_variants)[0]['selectedMainTranscriptId'], transcript_id) def test_update_variant_acmg_classification(self): update_variant_acmg_classification_url = reverse(update_variant_acmg_classification_handler, args=[VARIANT_GUID]) diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 7aa3f63934..6aaf14cb6a 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -19,7 +19,6 @@ from seqr.utils.middleware import ErrorsWarningsException from seqr.views.utils.json_utils import create_json_response from seqr.utils.logging_utils import SeqrLogger -from seqr.utils.xpos_utils import get_chrom_pos from seqr.views.utils.orm_to_json_utils import get_json_for_matchmaker_submissions, get_json_for_saved_variants,\ add_individual_hpo_details, INDIVIDUAL_DISPLAY_NAME_EXPR, AIP_TAG_TYPE from seqr.views.utils.permissions_utils import analyst_required, user_is_analyst, get_project_guids_user_can_view, \ @@ -63,7 +62,7 @@ def mme_details(request): response = { 'submissions': list(submissions_by_guid.values()), 'genesById': genes_by_id, - 'savedVariantsByGuid': {s['variantGuid']: {**s, 'chrom': get_chrom_pos(s['xpos'])[0] , 'pos': get_chrom_pos(s['xpos'])[1]} for s in saved_variants}, + 'savedVariantsByGuid': {s['variantGuid']: s for s in saved_variants}, } if user_is_analyst(request.user): response['metrics'] = get_mme_metrics() diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index 1d82998deb..8677bda356 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -27,7 +27,7 @@ } VARIANT_TAG_RESPONSE_KEYS = { - 'variantTagsByGuid', 'variantNotesByGuid', 'variantFunctionalDataByGuid', 'savedVariantsByGuid', + 'variantTagsByGuid', 'variantNotesByGuid', 'variantFunctionalDataByGuid', 'savedVariantsByGuid', 'variantsById', } SAVED_VARIANT_RESPONSE_KEYS = { *VARIANT_TAG_RESPONSE_KEYS, 'projectsByGuid', 'locusListsByGuid', 'genesById', 'totalSampleCounts', @@ -331,7 +331,7 @@ def test_mme_details(self, mock_datetime): {'ENSG00000240361', 'ENSG00000223972', 'ENSG00000135953'}) self.assertEqual(len(response_json['submissions']), self.NUM_MANAGER_SUBMISSIONS) self.assertSetEqual(set(next(iter(response_json['savedVariantsByGuid'].values())).keys()), { - 'chrom', 'pos', 'genomeVersion', *SAVED_VARIANT_FIELDS, + 'genomeVersion', *SAVED_VARIANT_FIELDS, }) # Test analyst behavior diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 09f4ccc04c..6f7013eb8f 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -12,7 +12,7 @@ from math import ceil import re -from reference_data.models import GENOME_VERSION_GRCh38 +from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_LOOKUP from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Sample from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, \ variant_lookup, export_variants @@ -91,8 +91,11 @@ def _get_or_create_results_model(search_hash, search_context, user): raise Exception('Invalid search hash: {}'.format(search_hash)) all_project_genome_version = _all_project_family_search_genome(search_context) + include_no_access_projects = search_context.get('includeNoAccessProjects') if all_project_genome_version: families = _all_genome_version_families(all_project_genome_version, user) + if not (families or include_no_access_projects): + raise Exception(f'No data available for genome version "{GENOME_VERSION_LOOKUP[all_project_genome_version]}"') elif search_context.get('projectFamilies'): all_families = set() for project_family in search_context['projectFamilies']: @@ -113,7 +116,7 @@ def _get_or_create_results_model(search_hash, search_context, user): search_dict = search_context.get('search', {}) if search_context.get('previousSearchHash') and (search_dict.get('exclude') or {}).get('previousSearch'): search_dict['exclude']['previousSearchHash'] = search_context['previousSearchHash'] - if search_context.get('includeNoAccessProjects'): + if include_no_access_projects: search_dict['no_access_project_genome_version'] = all_project_genome_version search_model = VariantSearch.objects.filter(search=search_dict).filter( Q(created_by=user) | Q(name__isnull=False)).first() @@ -148,7 +151,7 @@ def query_single_variant_handler(request, variant_id): def _process_variants(variants, families, request, add_all_context=False, add_locus_list_detail=False, genome_version=None): if not variants: - return {'searchedVariants': variants} + return {'searchedVariantIds': [], 'variantsById': {}} flat_variants = _flatten_variants(variants) variants_by_id = {v['variantId']: v for v in flat_variants} @@ -157,7 +160,9 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo response_json = get_variants_response( request, saved_variants, response_variants=flat_variants, add_all_context=add_all_context, add_locus_list_detail=add_locus_list_detail, genome_version=genome_version or families[0].project.genome_version) - response_json['searchedVariants'] = variants + response_json['searchedVariantIds'] = [ + [v['variantId'] for v in variant] if isinstance(variant, list) else variant['variantId'] for variant in variants + ] for saved_variant in response_json['savedVariantsByGuid'].values(): family_guids = saved_variant['familyGuids'] @@ -264,14 +269,14 @@ def export_variants_handler(request, search_hash): family_guid: saved_variant['variantGuid'] for family_guid in saved_variant['familyGuids'] } - if any(len(variant['familyGuids']) > MAX_FAMILIES_PER_ROW for variant in variants): + if any(len(variant.get('familyGuids', [])) > MAX_FAMILIES_PER_ROW for variant in variants): split_variants = [] for variant in variants: - if len(variant['familyGuids']) <= MAX_FAMILIES_PER_ROW: + if len(variant.get('familyGuids', [])) <= MAX_FAMILIES_PER_ROW: split_variants.append(variant) continue - num_split = ceil(len(variant['familyGuids']) / MAX_FAMILIES_PER_ROW) + num_split = ceil(len(variant.get('familyGuids', [])) / MAX_FAMILIES_PER_ROW) gens_per_row = ceil(len(variant['genotypes']) / num_split) gen_keys = sorted(variant['genotypes'].keys()) for i in range(num_split): @@ -283,15 +288,15 @@ def export_variants_handler(request, search_hash): variants = split_variants - max_families_per_variant = max([len(variant['familyGuids']) for variant in variants]) - max_samples_per_variant = max([len(variant['genotypes']) for variant in variants]) + max_families_per_variant = max([len(variant.get('familyGuids', [1])) for variant in variants]) + max_samples_per_variant = max([len(variant.get('genotypes', {})) for variant in variants]) rows = [] for variant in variants: row = [_get_field_value(variant, config) for config in VARIANT_EXPORT_DATA] family_saved_variants = saved_variants_by_variant_family.get(variant['variantId'], {}) - for family_guid in variant['familyGuids']: + for family_guid in variant.get('familyGuids', []): variant_guid = family_saved_variants.get(family_guid, '') family_tags = { 'family_id': family_ids_by_guid.get(family_guid), @@ -299,9 +304,13 @@ def export_variants_handler(request, search_hash): 'notes': [note for note in json_saved_variants['variantNotesByGuid'].values() if variant_guid in note['variantGuids']], } row += [_get_field_value(family_tags, config) for config in VARIANT_FAMILY_EXPORT_DATA] - row += ['' for i in range(len(VARIANT_FAMILY_EXPORT_DATA) * (max_families_per_variant - len(variant['familyGuids'])))] + num_blank_cols = len(VARIANT_FAMILY_EXPORT_DATA) * (max_families_per_variant - len(variant.get('familyGuids', []))) + if 'familyGuids' not in variant: + row.append(f"{variant['numFamilies']} Families") + num_blank_cols -= 1 + row += ['' for i in range(num_blank_cols)] - genotypes = [genotype for _, genotype in sorted(variant['genotypes'].items())] + genotypes = [genotype for _, genotype in sorted(variant.get('genotypes', {}).items())] for genotype in genotypes: row += [_get_field_value(genotype, config) for config in VARIANT_SAMPLE_DATA] row += ['' for i in range(len(VARIANT_SAMPLE_DATA) * (max_samples_per_variant - len(genotypes)))] @@ -537,7 +546,6 @@ def variant_lookup_handler(request): request, saved_variants=saved_variants, response_variants=variants, add_all_context=True, add_locus_list_detail=True, genome_version=genome_version, ) - response['variants'] = variants individual_guid_map = { (i['familyGuid'], i['individualId']): i['individualGuid'] for i in response['individualsByGuid'].values() diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 119b662664..631db9c59a 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -310,7 +310,7 @@ def test_vlm_lookup(self, mock_redis): expected_logs = [ ('VLM match request to Node 1', {'detail': expected_params}), ('VLM match request to Node 2', {'detail': expected_params}), - (f'VLM match error for Node 2: 400 Client Error: Bad Request for url: {node_2_url}', { + (f'VLM Node 2 match error: 400 Client Error: Bad Request for url: {node_2_url}', { 'severity': 'ERROR', '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', 'detail': expected_params, diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index c39e47b609..3f2c153c22 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -14,6 +14,7 @@ from reference_data.models import HumanPhenotypeOntology from seqr.models import GeneNote, VariantNote, VariantTag, VariantFunctionalData, SavedVariant, Family, CAN_VIEW, CAN_EDIT, \ get_audit_field_names, RnaSeqOutlier, RnaSeqSpliceOutlier +from seqr.utils.xpos_utils import get_chrom_pos from seqr.views.utils.json_utils import _to_camel_case from seqr.views.utils.permissions_utils import has_project_permissions, \ project_has_anvil, get_workspace_collaborator_perms, user_is_analyst, user_is_data_manager, user_is_pm, \ @@ -391,7 +392,7 @@ def get_json_for_analysis_group(analysis_group, **kwargs): return _get_json_for_model(analysis_group, get_json_for_models=get_json_for_analysis_groups, **kwargs) -def get_json_for_saved_variants(saved_variants, add_details=False, additional_model_fields=None, additional_values=None, genome_version=None): +def get_json_for_saved_variants(saved_variants, additional_model_fields=None, additional_values=None): sv_additional_values = { 'familyGuids': ArrayAgg('family__guid', distinct=True), } @@ -400,45 +401,17 @@ def get_json_for_saved_variants(saved_variants, add_details=False, additional_mo additional_fields = [] additional_fields += additional_model_fields or [] - if add_details: - additional_fields += ['saved_variant_json', 'key', 'genotypes', 'dataset_type'] + ( - [] if genome_version else ['family__project__genome_version'] - ) results = get_json_for_queryset( saved_variants, guid_key='variantGuid', additional_values=sv_additional_values, additional_model_fields=additional_fields, ) - if add_details: - _add_clickhouse_annotations(results, genome_version) - for result in results: - result.update({k: v for k, v in result.pop('savedVariantJson').items() if k not in result}) - - return results - - -def _add_clickhouse_annotations(results, genome_version): - from clickhouse_search.search import get_clickhouse_variant_annotations - results_by_genome_version_dataset_type = defaultdict(lambda: defaultdict(list)) for result in results: - dataset_type = result.pop('datasetType') - gv = genome_version or result.pop('familyProjectGenomeVersion') - if result['key']: - results_by_genome_version_dataset_type[gv][dataset_type].append(result) - else: - result.pop('genotypes') + chrom, pos = get_chrom_pos(result['xpos']) + result.update({'chrom': chrom, 'pos': pos}) - for gv, grouped_results in results_by_genome_version_dataset_type.items(): - for dataset_type, gv_results in grouped_results.items(): - keys = {result['key'] for result in gv_results} - annotations_by_key = { - annotations['key']: {k: v for k, v in annotations.items() if k not in gv_results[0]} - for annotations in get_clickhouse_variant_annotations(gv, dataset_type, keys) - } - for result in gv_results: - if result['key']: - result.update(annotations_by_key[result['key']]) + return results def _format_functional_tags(tags): @@ -501,10 +474,10 @@ def get_json_for_saved_variants_child_entities(tag_cls, saved_variant_id_map, ta return tags, variant_tag_map -def get_json_for_saved_variants_with_tags(saved_variants, additional_model_fields=None, **kwargs): +def get_json_for_saved_variants_with_tags(saved_variants, additional_model_fields=None): variants_by_guid = { variant['variantGuid']: dict(tagGuids=[], functionalDataGuids=[], noteGuids=[], **variant) - for variant in get_json_for_saved_variants(saved_variants, additional_model_fields=(additional_model_fields or []) + ['id'], **kwargs) + for variant in get_json_for_saved_variants(saved_variants, additional_model_fields=(additional_model_fields or []) + ['id']) } saved_variant_id_map = {} diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index 1be9e8a56a..a8af246857 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -712,9 +712,10 @@ def _get_list_param(call, param): 'projectGuid', 'familyGuid', 'individualGuid', 'sampleGuid', 'filePath', 'indexFilePath', 'sampleId', 'sampleType', } -SAVED_VARIANT_FIELDS = {'variantGuid', 'variantId', 'familyGuids', 'xpos', 'ref', 'alt', 'selectedMainTranscriptId', 'acmgClassification'} +SAVED_VARIANT_FIELDS = {'variantGuid', 'variantId', 'familyGuids', 'xpos', 'ref', 'alt', 'selectedMainTranscriptId', 'acmgClassification', + 'chrom', 'pos', 'key', 'geneIds', 'genotypes', 'svType'} SAVED_VARIANT_DETAIL_FIELDS = { - 'chrom', 'pos', 'genomeVersion', 'liftedOverGenomeVersion', 'liftedOverChrom', 'liftedOverPos', 'tagGuids', + 'genomeVersion', 'liftedOverGenomeVersion', 'liftedOverChrom', 'liftedOverPos', 'tagGuids', 'functionalDataGuids', 'noteGuids', 'genotypes', 'hgmd', 'CAID', 'transcripts', 'populations', 'predictions', 'rsid', 'clinvar', 'acmgClassification' } diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 7a3aed8dc1..eab4d543b2 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -8,7 +8,7 @@ import redis from clickhouse_search.backend.functions import ArrayDistinct, ArrayMap -from clickhouse_search.search import get_variants_queryset +from clickhouse_search.search import get_variants_queryset, get_clickhouse_variant_annotations from matchmaker.models import MatchmakerSubmissionGenes, MatchmakerSubmission from reference_data.models import TranscriptInfo, Omim, GENOME_VERSION_GRCh38 from seqr.models import SavedVariant, VariantSearchResults, Family, LocusList, LocusListInterval, LocusListGene, \ @@ -373,13 +373,24 @@ def _add_pa_detail(locus_list_gene, locus_list_guid, gene_json): def get_variants_response(request, saved_variants, response_variants=None, add_all_context=False, include_igv=True, add_locus_list_detail=False, include_individual_gene_scores=True, include_project_name=False, genome_version=None): - response = get_json_for_saved_variants_with_tags(saved_variants, add_details=True, genome_version=genome_version) \ + additional_model_fields = [] + if response_variants is None: + additional_model_fields = ['dataset_type', 'saved_variant_json'] + if not genome_version: + additional_model_fields.append('family__project__genome_version') + response = get_json_for_saved_variants_with_tags(saved_variants, additional_model_fields=additional_model_fields) \ if saved_variants is not None else {'savedVariantsByGuid': {}} - variants = list(response['savedVariantsByGuid'].values()) if response_variants is None else response_variants - if not variants: + if response_variants is None: + variants_by_id = _get_clickhouse_variant_annotations(response['savedVariantsByGuid'].values(), genome_version) + else: + variants_by_id = {variant['variantId']: variant for variant in response_variants} + + response['variantsById'] = variants_by_id + if not variants_by_id: return response + variants = variants_by_id.values() family_guids = {family_guid for variant in variants for family_guid in variant.get('familyGuids', [])} projects = Project.objects.filter(family__guid__in=family_guids).distinct() project = list(projects)[0] if len(projects) == 1 else None @@ -391,7 +402,7 @@ def get_variants_response(request, saved_variants, response_variants=None, add_a discovery_tags = None is_analyst = user_is_analyst(request.user) if is_analyst: - discovery_tags, discovery_response = get_json_for_discovery_tags(response['savedVariantsByGuid'].values(), request.user) + discovery_tags, discovery_response = get_json_for_discovery_tags(variants, request.user) response.update(discovery_response) response['locusListsByGuid'] = _add_locus_lists( @@ -430,6 +441,30 @@ def get_variants_response(request, saved_variants, response_variants=None, add_a return response +def _get_clickhouse_variant_annotations(variants, genome_version): + variant_keys_by_genome_version_dataset_type = defaultdict(lambda: defaultdict(set)) + variants_by_id = defaultdict(dict) + for variant in variants: + dataset_type = variant.pop('datasetType') + gv = genome_version or variant.pop('familyProjectGenomeVersion') + variant_json = variant.pop('savedVariantJson') + variants_by_id[variant['variantId']] = { + **variant_json, + **variants_by_id[variant['variantId']], + **variant, + 'familyGuids': variant['familyGuids'] + variants_by_id[variant['variantId']].get('familyGuids', []), + 'genotypes': {**variant['genotypes'], **variants_by_id[variant['variantId']].get('genotypes', {})}, + } + if variant['key']: + variant_keys_by_genome_version_dataset_type[gv][dataset_type].add(variant['key']) + + for gv, gv_keys in variant_keys_by_genome_version_dataset_type.items(): + for dataset_type, keys in gv_keys.items(): + for annotations in get_clickhouse_variant_annotations(gv, dataset_type, keys): + variants_by_id[annotations['variantId']].update(annotations) + + return variants_by_id + def _mme_response_context(saved_variants_by_guid): mme_submission_genes = MatchmakerSubmissionGenes.objects.filter( saved_variant__guid__in=saved_variants_by_guid.keys()).values( diff --git a/seqr/views/utils/vlm_utils.py b/seqr/views/utils/vlm_utils.py index cb32ff6e1f..658e4ba79a 100644 --- a/seqr/views/utils/vlm_utils.py +++ b/seqr/views/utils/vlm_utils.py @@ -50,7 +50,7 @@ def vlm_lookup(user, chrom, pos, ref, alt, genome_version=None, **kwargs): result_id = parsed_id[0] if len(parsed_id) == 2 else next(iter(results[client_name].keys())) results[client_name][result_id]['counts'][count_type] = result['resultsCount'] except Exception as e: - logger.error(f'VLM match error for {client_name}: {e}', user, detail=params) + logger.error(f'VLM {client_name} match error: {e}', user, detail=params) return results diff --git a/ui/package-lock.json b/ui/package-lock.json index b6edfb300b..f58320ac31 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -105,7 +105,7 @@ "mini-css-extract-plugin": "^2.3.0", "object-assign": "4.1.1", "patch-package": "^8.0.1", - "postcss": "^8.4.31", + "postcss": "^8.5.10", "postcss-flexbugs-fixes": "^5.0.2", "postcss-loader": "^6.1.1", "promise": "8.0.1", @@ -12996,9 +12996,9 @@ "integrity": "sha512-xGPXiFVl4YED9Jh7Euv2V220mriG9u4B2TA6Ybjc1catrstKD2PpIdU3U0RKpkVBC2EhmL/F0sPCr9vrFTNRag==" }, "node_modules/nanoid": { - "version": "3.3.8", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", - "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", "funding": [ { "type": "github", @@ -13859,9 +13859,9 @@ "integrity": "sha512-+G+EkOPoE5S/zChTpmBSSDYmhXJ5PsW8eMhH8cP/CQHMFPBG/kC9Y5IIw6qNYgdJ+/COf0ddY2li28iHaZRSjw==" }, "node_modules/postcss": { - "version": "8.4.31", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", - "integrity": "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==", + "version": "8.5.10", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.10.tgz", + "integrity": "sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==", "funding": [ { "type": "opencollective", @@ -13877,9 +13877,9 @@ } ], "dependencies": { - "nanoid": "^3.3.6", - "picocolors": "^1.0.0", - "source-map-js": "^1.0.2" + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" }, "engines": { "node": "^10 || ^12 || >=14" @@ -16242,9 +16242,9 @@ } }, "node_modules/source-map-js": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", - "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", "engines": { "node": ">=0.10.0" } @@ -29443,9 +29443,9 @@ } }, "nanoid": { - "version": "3.3.8", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", - "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==" + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==" }, "natural-compare": { "version": "1.4.0", @@ -30114,13 +30114,13 @@ "integrity": "sha512-+G+EkOPoE5S/zChTpmBSSDYmhXJ5PsW8eMhH8cP/CQHMFPBG/kC9Y5IIw6qNYgdJ+/COf0ddY2li28iHaZRSjw==" }, "postcss": { - "version": "8.4.31", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", - "integrity": "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==", + "version": "8.5.10", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.10.tgz", + "integrity": "sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==", "requires": { - "nanoid": "^3.3.6", - "picocolors": "^1.0.0", - "source-map-js": "^1.0.2" + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" } }, "postcss-calc": { @@ -32010,9 +32010,9 @@ "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" }, "source-map-js": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", - "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==" }, "source-map-support": { "version": "0.5.21", diff --git a/ui/package.json b/ui/package.json index b8943d2bf4..0e254b1766 100644 --- a/ui/package.json +++ b/ui/package.json @@ -49,7 +49,7 @@ "mini-css-extract-plugin": "^2.3.0", "object-assign": "4.1.1", "patch-package": "^8.0.1", - "postcss": "^8.4.31", + "postcss": "^8.5.10", "postcss-flexbugs-fixes": "^5.0.2", "postcss-loader": "^6.1.1", "promise": "8.0.1", diff --git a/ui/pages/Project/fixtures.js b/ui/pages/Project/fixtures.js index 29a4152ea8..033fc74b57 100644 --- a/ui/pages/Project/fixtures.js +++ b/ui/pages/Project/fixtures.js @@ -421,31 +421,245 @@ export const STATE_WITH_2_FAMILIES = { savedVariantsByGuid: { SV0000004_116042722_r0390_1000: { alt: "T", - annotation: { - cadd_phred: "27.2", - freqs: { AF: null, exac: 0.0006726888333653661, g1k: 0, gnomad_exomes: 0.00006505916317651364 }, - popCounts: { AC: null, AN: null, exac_hemi: null, exac_hom: null, gnomadExomesAC: null, gnomadGenomesAC: null} , - vepAnnotations: [ - { - aminoAcids: "P/X", - canonical: "YES", - cdnaPosition: "897", - cdsPosition: "859", - codons: "Ccc/cc", - consequence: "frameshift_variant", - hgvsc: "ENST00000456743.1:c.862delC", - hgvsp: "ENSP00000389625.1:p.Leu288SerfsTer10", - isChosenTranscript: true, - transcriptId: "ENST00000456743", - } - ], - vepConsequence: "frameshift_variant", - vepGroup: "frameshift", - worstVepAnnotation: { - aminoAcids: "P/X", hgvsc: "ENST00000456743.1:c.862delC", hgvsp: "ENSP00000389625.1:p.Leu288SerfsTer10", - lof: "HC", lofFilter: "", lofFlags: "SINGLE_EXON", proteinPosition: "287", symbol: "OR2M3" - } + chrom: "22", + clinvar: { clinsig: "", variantId: null }, + familyGuids: ["F011652_1"], + functionalDataGuids: [], + genomeVersion: "37", + genotypes: { + I021475_na19675_1: { + ab: 1, + ad: "0,74", + alleles: ["T", "T"], + cnvs: {LRR_median: null, LRR_sd: null, array: null, caller: null, cn: null, freq: null, size: null}, + dp: "74", + filter: "pass", + gq: 99, + numAlt: 2, + pl: "358,132,0", + }, + NA19678: { + ab: 0, + ad: "77,0", + alleles: ["TC", "TC"], + cnvs: {LRR_median: null, LRR_sd: null, array: null, caller: null, cn: null, freq: null, size: null}, + dp: "77", + filter: "pass", + gq: 99, + numAlt: 0, + pl: "0,232,3036", + }, + NA19679: { + ab: 0, + ad: "71,0", + alleles: ["TC", "TC"], + cnvs: {LRR_median: null, LRR_sd: null, array: null, caller: null, cn: null, freq: null, size: null}, + dp: "71", + filter: "pass", + gq: 99, + numAlt: 0, + pl: "0,213,1918", + }, + }, + geneIds: ['ENSG00000228198'], + noteGuids: ['VN0727076_116042722_r0390_1000'], + pos: 45919065, + ref: "TTTC", + tagGuids: ['VT1726942_1248367227_r0390_101'], + selectedMainTranscript: { + aminoAcids: "P/X", hgvsc: "ENST00000456743.1:c.862delC", hgvsp: "ENSP00000389625.1:p.Leu288SerfsTer10", lof: "HC", + lofFilter: "", lofFlags: "SINGLE_EXON", proteinPosition: "287", symbol: "OR2M3", geneId: 'ENSG00000228198', + majorConsequence: 'frameshift_variant', transcriptId: 'ENST00000456743' + }, + variantId: "22-45919065-TTTC-T", + variantGuid: "SV0000004_116042722_r0390_1000", + xpos: 22045919065, + }, + SV0000002_1248367227_r0390_100: { + alt: "T", + chrom: "1", + familyGuids: ["F011652_1"], + functionalDataGuids: ['FD_248367227_r0390_100_1', 'FD_248367227_r0390_100_2'], + genomeVersion: "37", + genotypes: { + I021475_na19675_1: { + ab: 1, + ad: "0,74", + alleles: ["T", "T"], + cnvs: {LRR_median: null, LRR_sd: null, array: null, caller: null, cn: null, freq: null, size: null}, + dp: "74", + filter: "pass", + gq: 99, + numAlt: 2, + pl: "358,132,0", + }, + NA19678: { + ab: 0, + ad: "77,0", + alleles: ["TC", "TC"], + cnvs: {LRR_median: null, LRR_sd: null, array: null, caller: null, cn: null, freq: null, size: null}, + dp: "77", + filter: "pass", + gq: 99, + numAlt: 0, + pl: "0,232,3036", + }, + NA19679: { + ab: 0, + ad: "71,0", + alleles: ["TC", "TC"], + cnvs: {LRR_median: null, LRR_sd: null, array: null, caller: null, cn: null, freq: null, size: null}, + dp: "71", + filter: "pass", + gq: 99, + numAlt: 0, + pl: "0,213,1918", + }, + }, + noteGuids: [], + geneIds: ['ENSG00000228198'], + pos: 248367227, + ref: "TC", + tagGuids: ["VT1726942_1248367227_r0390_100", "VT1708635_1248367227_r0390_100"], + variantId: "1-248367227-TC-T", + variantGuid: "SV0000002_1248367227_r0390_100", + xpos: 1248367227, + }, + SV0000002_SV48367227_r0390_100: { + alt: null, + chrom: "1", + familyGuids: ["F011652_1"], + functionalDataGuids: [], + genomeVersion: "37", + geneIds: ['ENSG00000228198', 'ENSG00000164458'], + genotypes: { + I021475_na19675_1: { + cn: 0, + qs: 57, + numAlt: -1, + }, + NA19678: { + cn: 2, + numAlt: -1, + }, + NA19679: { + cn: 2, + numAlt: -1, + }, + }, + noteGuids: [], + pos: 248367227, + end: 248369100, + ref: null, + tagGuids: [], + variantId: "batch_123_DEL", + variantGuid: "SV0000002_SV48367227_r0390_100", + xpos: 1248367227, + }, + SV0000003_2246859832_r0390_100: { + alt: "T", + chrom: "22", + familyGuids: ["F011652_2"], + functionalDataGuids: [], + genomeVersion: "37", + genotypes: { + I021475_na19675_1: { + ab: 1, + ad: "0,74", + alleles: ["T", "T"], + cnvs: {LRR_median: null, LRR_sd: null, array: null, caller: null, cn: null, freq: null, size: null}, + dp: "74", + filter: "pass", + gq: 99, + numAlt: 2, + pl: "358,132,0", + }, + NA19678: { + ab: 0, + ad: "77,0", + alleles: ["TC", "TC"], + cnvs: {LRR_median: null, LRR_sd: null, array: null, caller: null, cn: null, freq: null, size: null}, + dp: "77", + filter: "pass", + gq: 99, + numAlt: 0, + pl: "0,232,3036", + }, + NA19679: { + ab: 0, + ad: "71,0", + alleles: ["TC", "TC"], + cnvs: {LRR_median: null, LRR_sd: null, array: null, caller: null, cn: null, freq: null, size: null}, + dp: "71", + filter: "pass", + gq: 99, + numAlt: 0, + pl: "0,213,1918", + }, }, + noteGuids: [], + origAltAlleles: ["T"], + pos: 248367227, + ref: "C", + tagGuids: ['VT1726942_1248367227_r0390_102'], + variantId: "22-248367227-C-T", + variantGuid: "SV0000003_2246859832_r0390_100", + xpos: 22046859832, + }, + SV0000005_2246859833_r0390_100: { + alt: "T", + chrom: "22", + familyGuids: ["F011652_2"], + functionalDataGuids: [], + genomeVersion: "37", + genotypes: { + I021475_na19675_1: { + ab: 1, + ad: "0,74", + alleles: ["T", "T"], + cnvs: {LRR_median: null, LRR_sd: null, array: null, caller: null, cn: null, freq: null, size: null}, + dp: "74", + filter: "pass", + gq: 99, + numAlt: 2, + pl: "358,132,0", + }, + NA19678: { + ab: 0, + ad: "77,0", + alleles: ["TC", "TC"], + cnvs: {LRR_median: null, LRR_sd: null, array: null, caller: null, cn: null, freq: null, size: null}, + dp: "77", + filter: "pass", + gq: 99, + numAlt: 0, + pl: "0,232,3036", + }, + NA19679: { + ab: 0, + ad: "71,0", + alleles: ["TC", "TC"], + cnvs: {LRR_median: null, LRR_sd: null, array: null, caller: null, cn: null, freq: null, size: null}, + dp: "71", + filter: "pass", + gq: 99, + numAlt: 0, + pl: "0,213,1918", + }, + }, + noteGuids: [], + pos: 248367228, + projectGuid: 'R0237_1000_genomes_demo', + ref: "C", + tagGuids: ['VT1726942_1248367227_r0390_102'], + variantId: "22-248367228-C-T", + variantGuid: "SV0000005_2246859833_r0390_100", + xpos: 22046859833, + }, + }, + variantsById: { + "22-45919065-TTTC-T": { + alt: "T", chrom: "22", clinvar: { clinsig: "", variantId: null }, familyGuids: ["F011652_1"], @@ -506,49 +720,12 @@ export const STATE_WITH_2_FAMILIES = { variantGuid: "SV0000004_116042722_r0390_1000", xpos: 22045919065, }, - SV0000002_1248367227_r0390_100: { + "1-248367227-TC-T": { alt: "T", - annotation: { - cadd_phred: "27.2", - freqs: { AF: null, exac: 0.0006726888333653661, g1k: 0, gnomad_exomes: 0.00006505916317651364 }, - popCounts: { AC: null, AN: null, exac_hemi: null, exac_hom: null, gnomadExomesAC: null, gnomadGenomesAC: null} , - vepAnnotations: [ - { - aminoAcids: "P/X", - canonical: "YES", - cdnaPosition: "897", - cdsPosition: "859", - codons: "Ccc/cc", - consequence: "frameshift_variant", - hgvsc: "ENST00000456743.1:c.862delC", - hgvsp: "ENSP00000389625.1:p.Leu288SerfsTer10", - isChosenTranscript: true, - transcriptId: "ENST00000456743", - } - ], - vepConsequence: "frameshift_variant", - vepGroup: "frameshift", - worstVepAnnotation: { - aminoAcids: "P/X", hgvsc: "ENST00000456743.1:c.862delC", hgvsp: "ENSP00000389625.1:p.Leu288SerfsTer10", - lof: "HC", lofFilter: "", lofFlags: "SINGLE_EXON", proteinPosition: "287", symbol: "OR2M3" - } - }, chrom: "1", clinvar: { clinsig: "", variantId: null }, familyGuids: ["F011652_1"], functionalDataGuids: ['FD_248367227_r0390_100_1', 'FD_248367227_r0390_100_2'], - genes: [ - { - constraints: { - lof: { constraint: 0.0671997116609769, rank: 8248, totalGenes: 18225 }, - missense: { constraint: -0.7885573790993861, rank: 15052, totalGenes: 18225 }, - }, - diseaseDbPhenotypes: [], - diseaseGeneLists: [], - geneId: "ENSG00000228198", - symbol: "OR2M3", - } - ], genomeVersion: "37", genotypes: { I021475_na19675_1: { @@ -594,14 +771,13 @@ export const STATE_WITH_2_FAMILIES = { mainTranscriptId: 'ENST00000262738', transcripts: {ENSG00000228198: [{transcriptId: 'ENST00000262738', majorConsequence: 'missense_variant'}]}, pos: 248367227, - projectGuid: 'R0237_1000_genomes_demo', ref: "TC", tagGuids: ["VT1726942_1248367227_r0390_100", "VT1708635_1248367227_r0390_100"], variantId: "1-248367227-TC-T", variantGuid: "SV0000002_1248367227_r0390_100", xpos: 1248367227, }, - SV0000002_SV48367227_r0390_100: { + "batch_123_DEL": { alt: null, chrom: "1", familyGuids: ["F011652_1"], @@ -657,49 +833,12 @@ export const STATE_WITH_2_FAMILIES = { variantGuid: "SV0000002_SV48367227_r0390_100", xpos: 1248367227, }, - SV0000003_2246859832_r0390_100: { + "22-248367227-C-T": { alt: "T", - annotation: { - cadd_phred: "27.2", - freqs: { AF: null, exac: 0.0006726888333653661, g1k: 0, gnomad_exomes: 0.00006505916317651364 }, - popCounts: { AC: null, AN: null, exac_hemi: null, exac_hom: null, gnomadExomesAC: null, gnomadGenomesAC: null} , - vepAnnotations: [ - { - aminoAcids: "P/X", - canonical: "YES", - cdnaPosition: "897", - cdsPosition: "859", - codons: "Ccc/cc", - consequence: "frameshift_variant", - hgvsc: "ENST00000456743.1:c.862delC", - hgvsp: "ENSP00000389625.1:p.Leu288SerfsTer10", - isChosenTranscript: true, - transcriptId: "ENST00000456743", - } - ], - vepConsequence: "frameshift_variant", - vepGroup: "frameshift", - worstVepAnnotation: { - aminoAcids: "P/X", hgvsc: "ENST00000456743.1:c.862delC", hgvsp: "ENSP00000389625.1:p.Leu288SerfsTer10", - lof: "HC", lofFilter: "", lofFlags: "SINGLE_EXON", proteinPosition: "287", symbol: "OR2M3" - } - }, chrom: "22", clinvar: { clinsig: "", variantId: null }, familyGuids: ["F011652_2"], functionalDataGuids: [], - genes: [ - { - constraints: { - lof: { constraint: 0.0671997116609769, rank: 8248, totalGenes: 18225 }, - missense: { constraint: -0.7885573790993861, rank: 15052, totalGenes: 18225 }, - }, - diseaseDbPhenotypes: [], - diseaseGeneLists: [], - geneId: "ENSG00000228198", - symbol: "OR2M3", - } - ], genomeVersion: "37", genotypes: { I021475_na19675_1: { @@ -752,49 +891,12 @@ export const STATE_WITH_2_FAMILIES = { variantGuid: "SV0000003_2246859832_r0390_100", xpos: 22046859832, }, - SV0000005_2246859833_r0390_100: { + "22-248367228-C-T": { alt: "T", - annotation: { - cadd_phred: "27.2", - freqs: { AF: null, exac: 0.0006726888333653661, g1k: 0, gnomad_exomes: 0.00006505916317651364 }, - popCounts: { AC: null, AN: null, exac_hemi: null, exac_hom: null, gnomadExomesAC: null, gnomadGenomesAC: null} , - vepAnnotations: [ - { - aminoAcids: "P/X", - canonical: "YES", - cdnaPosition: "897", - cdsPosition: "859", - codons: "Ccc/cc", - consequence: "frameshift_variant", - hgvsc: "ENST00000456743.1:c.862delC", - hgvsp: "ENSP00000389625.1:p.Leu288SerfsTer10", - isChosenTranscript: true, - transcriptId: "ENST00000456743", - } - ], - vepConsequence: "frameshift_variant", - vepGroup: "frameshift", - worstVepAnnotation: { - aminoAcids: "P/X", hgvsc: "ENST00000456743.1:c.862delC", hgvsp: "ENSP00000389625.1:p.Leu288SerfsTer10", - lof: "HC", lofFilter: "", lofFlags: "SINGLE_EXON", proteinPosition: "287", symbol: "OR2M3" - } - }, chrom: "22", clinvar: { clinsig: "", variantId: null }, familyGuids: ["F011652_2"], functionalDataGuids: [], - genes: [ - { - constraints: { - lof: { constraint: 0.0671997116609769, rank: 8248, totalGenes: 18225 }, - missense: { constraint: -0.7885573790993861, rank: 15052, totalGenes: 18225 }, - }, - diseaseDbPhenotypes: [], - diseaseGeneLists: [], - geneId: "ENSG00000228198", - symbol: "OR2M3", - } - ], genomeVersion: "37", genotypes: { I021475_na19675_1: { diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index 7fb9dafb6b..7b5b560511 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -216,7 +216,7 @@ export const getTaggedVariantsByFamily = createSelector( ).reduce((acc, variant) => { const { familyGuids, ...variantDetail } = variant variantDetail.tags = variant.tagGuids.map(tagGuid => variantTagsByGuid[tagGuid]) - variantDetail.genes = Object.keys(variant.transcripts || {}).map(geneId => genesById[geneId]) + variantDetail.genes = (variant.geneIds || []).map(geneId => genesById[geneId]) familyGuids.forEach((familyGuid) => { if (!acc[familyGuid]) { acc[familyGuid] = [] diff --git a/ui/pages/Search/VariantLookup.jsx b/ui/pages/Search/VariantLookup.jsx index e47264c8ba..4b9145a4bf 100644 --- a/ui/pages/Search/VariantLookup.jsx +++ b/ui/pages/Search/VariantLookup.jsx @@ -242,12 +242,15 @@ const mapVariantStateToProps = (state, ownProps) => ({ const LookupVariant = connect(mapVariantStateToProps)(BaseLookupVariant) -const VariantDisplay = ({ variants }) => ( - (variants || [])[0]?.lookupFamilyGuids ? : -) +const VariantDisplay = ({ variantsById }) => { + const variants = Object.values(variantsById || {}) + return variants[0]?.lookupFamilyGuids ? + : + +} VariantDisplay.propTypes = { - variants: PropTypes.arrayOf(PropTypes.object), + variantsById: PropTypes.object, } const onQueryParamSubmit = (updateQueryParams, data) => { diff --git a/ui/pages/Search/reducers.js b/ui/pages/Search/reducers.js index 67cbd5ea9a..a68c73498d 100644 --- a/ui/pages/Search/reducers.js +++ b/ui/pages/Search/reducers.js @@ -127,7 +127,7 @@ export const loadSingleSearchedVariant = ({ variantId, familyGuid }) => (dispatc return new HttpRequestHelper(url, (responseJson) => { dispatch({ type: RECEIVE_DATA, updatesById: responseJson }) - dispatch({ type: RECEIVE_SEARCHED_VARIANTS, newValue: responseJson.searchedVariants }) + dispatch({ type: RECEIVE_SEARCHED_VARIANTS, newValue: responseJson.searchedVariantIds }) }, (e) => { dispatch({ type: RECEIVE_SEARCHED_VARIANTS, error: e.message, newValue: [] }) @@ -194,7 +194,7 @@ export const loadSearchedVariants = ( new HttpRequestHelper(url, (responseJson) => { dispatch({ type: RECEIVE_DATA, updatesById: responseJson }) - dispatch({ type: RECEIVE_SEARCHED_VARIANTS, newValue: responseJson.searchedVariants }) + dispatch({ type: RECEIVE_SEARCHED_VARIANTS, newValue: responseJson.searchedVariantIds }) dispatch({ type: RECEIVE_SAVED_SEARCHES, updatesById: { searchesByHash: { [searchHash]: responseJson.search } } }) }, (e) => { @@ -231,7 +231,7 @@ export const reducers = { REQUEST_MULTI_PROJECT_SEARCH_CONTEXT, RECEIVE_MULTI_PROJECT_SEARCH_CONTEXT, ), flattenCompoundHet: createSingleObjectReducer(UPDATE_COMPOUND_HET_DISPLAY), - searchedVariants: createSingleValueReducer(RECEIVE_SEARCHED_VARIANTS, []), + searchedVariantIds: createSingleValueReducer(RECEIVE_SEARCHED_VARIANTS, []), searchedVariantsLoading: loadingReducer(REQUEST_SEARCHED_VARIANTS, RECEIVE_SEARCHED_VARIANTS), searchGeneBreakdown: createObjectsByIdReducer(RECEIVE_SEARCH_GENE_BREAKDOWN, 'searchGeneBreakdown'), searchGeneBreakdownLoading: loadingReducer(REQUEST_SEARCH_GENE_BREAKDOWN, RECEIVE_SEARCH_GENE_BREAKDOWN), diff --git a/ui/pages/Search/selectors.js b/ui/pages/Search/selectors.js index 6fea91ec0d..27930e368f 100644 --- a/ui/pages/Search/selectors.js +++ b/ui/pages/Search/selectors.js @@ -15,6 +15,7 @@ import { getProjectDatasetTypes, getActiveDatasetsByFamily, getSortedIndividualsByFamily, + getVariantsById, } from 'redux/selectors' import { getVariantMainGeneId, getVariantSummary, FAMILY_ANALYSIS_STATUS_LOOKUP } from 'shared/utils/constants' import { compareObjects } from 'shared/utils/sortUtils' @@ -26,7 +27,7 @@ export const getSavedSearchesByGuid = state => state.savedSearchesByGuid export const getSavedSearchesIsLoading = state => state.savedSearchesLoading.isLoading export const getSavedSearchesLoadingError = state => state.savedSearchesLoading.errorMessage export const getFlattenCompoundHet = state => state.flattenCompoundHet -export const getSearchedVariants = state => state.searchedVariants +const getSearchedVariantIds = state => state.searchedVariantIds export const getSearchedVariantsIsLoading = state => state.searchedVariantsLoading.isLoading export const getSearchedVariantsErrorMessage = state => state.searchedVariantsLoading.errorMessage export const getSearchGeneBreakdown = state => state.searchGeneBreakdown @@ -246,8 +247,12 @@ export const getAnalysisGroupOptions = createSelector( export const getDisplayVariants = createSelector( getFlattenCompoundHet, - getSearchedVariants, - (flattenCompoundHet, searchedVariants) => { + getSearchedVariantIds, + getVariantsById, + (flattenCompoundHet, searchedVariantIds, variantsById) => { + const searchedVariants = (searchedVariantIds || []).map( + variantId => (Array.isArray(variantId) ? variantId.map(vId => variantsById[vId]) : variantsById[variantId]), + ) const shouldFlatten = Object.values(flattenCompoundHet || {}).some(val => val) if (!shouldFlatten) { return searchedVariants || [] diff --git a/ui/redux/rootReducer.js b/ui/redux/rootReducer.js index d28acefe1f..99bf5f2136 100644 --- a/ui/redux/rootReducer.js +++ b/ui/redux/rootReducer.js @@ -283,6 +283,7 @@ const rootReducer = combineReducers({ variantTagsByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'variantTagsByGuid'), variantNotesByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'variantNotesByGuid'), variantFunctionalDataByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'variantFunctionalDataByGuid'), + variantsById: createObjectsByIdReducer(RECEIVE_DATA, 'variantsById'), searchesByHash: createObjectsByIdReducer(RECEIVE_SAVED_SEARCHES, 'searchesByHash'), searchFamiliesByHash: createObjectsByIdReducer(RECEIVE_SAVED_SEARCHES, 'searchFamiliesByHash'), transcriptsById: createObjectsByIdReducer(RECEIVE_DATA, 'transcriptsById'), diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index 93918f9f33..97ef046a96 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -19,6 +19,7 @@ export const getSavedVariantsByGuid = state => state.savedVariantsByGuid export const getVariantTagsByGuid = state => state.variantTagsByGuid export const getVariantNotesByGuid = state => state.variantNotesByGuid export const getVariantFunctionalDataByGuid = state => state.variantFunctionalDataByGuid +export const getVariantsById = state => state.variantsById export const getMmeSubmissionsByGuid = state => state.mmeSubmissionsByGuid export const getMmeResultsByGuid = state => state.mmeResultsByGuid export const getGenesById = state => state.genesById diff --git a/ui/shared/components/panel/fixtures.js b/ui/shared/components/panel/fixtures.js index 7765dfd8d5..4bca6989a7 100644 --- a/ui/shared/components/panel/fixtures.js +++ b/ui/shared/components/panel/fixtures.js @@ -12,14 +12,11 @@ export const USER = { username: 'test', } -export const VARIANT = { +const SAVED_VARIANT = { alt: "T", - mainTranscriptId: 'ENST00000456743', chrom: "1", - clinvar: { clinsig: "", variantId: null }, familyGuids: ["F011652_1"], functionalDataGuids: ['VFD0000002_1248367227_r0390_100', 'VFD0000001_1248367227_r0390_100'], - genomeVersion: "37", geneIds: ['ENSG00000228198'], genotypes: { NA19675: [{ @@ -64,14 +61,67 @@ export const VARIANT = { pl: "0,213,1918", }, }, + noteGuids: [], + pos: 248367227, // NOPMD + ref: "TC", + tagGuids: ['VT1708635_1248367227_r0390_100', 'VT1726942_1248367227_r0390_100'], + variantId: "1-248367227-TC-T", + variantGuid: "SV0000002_1248367227_r0390_100", + xpos: 1248367227, // NOPMD +} +export const VARIANT = { + ...SAVED_VARIANT, + mainTranscriptId: 'ENST00000456743', + clinvar: { clinsig: "", variantId: null }, + genomeVersion: "37", + genotypes: { + NA19675: [{ + ab: 1, + ad: "0,74", + alleles: ["T", "T"], + dp: "74", + filter: "pass", + gq: 99, + numAlt: 2, + pl: "358,132,0", + sampleType: "WES", + }, { + ab: 1, + ad: "0,74", + alleles: ["T", "T"], + dp: "74", + filter: "pass", + gq: 99, + numAlt: 2, + pl: "358,132,0", + sampleType: "WGS", + }], + NA19678: { + ab: 0, + ad: "77,0", + alleles: ["TC", "TC"], + dp: "77", + filter: "pass", + gq: 99, + numAlt: 0, + pl: "0,232,3036", + }, + NA19679: { + ab: 0, + ad: "71,0", + alleles: ["TC", "TC"], + dp: "71", + filter: "pass", + gq: 99, + numAlt: 0, + pl: "0,213,1918", + }, + }, hgmd: {accession: null, class: null}, liftedOverChrom: "", liftedOverGenomeVersion: "38", liftedOverPos: "", - noteGuids: [], origAltAlleles: ["T"], - projectGuid: 'R0237_1000_genomes_demo', - pos: 248367227, populations: { callset: { af: 0.03, ac: 7, an: 1032 }, g1k: { af: 0 }, @@ -81,8 +131,6 @@ export const VARIANT = { topmed: {}, }, predictions: { cadd: '27.2' }, - ref: "TC", - tagGuids: ['VT1708635_1248367227_r0390_100', 'VT1726942_1248367227_r0390_100'], transcripts: { ENSG00000228198: [ { @@ -99,9 +147,6 @@ export const VARIANT = { } ], }, - variantId: "SV0000002_1248367227_r0390_100", - variantGuid: "SV0000002_1248367227_r0390_100", - xpos: 1248367227, } export const SV_VARIANT = { @@ -383,7 +428,8 @@ export const STATE1 = { }, genesById: { 'ENSG00000228198': GENE }, genesLoading: {}, - savedVariantsByGuid: { SV0000002_1248367227_r0390_100: VARIANT }, + savedVariantsByGuid: { SV0000002_1248367227_r0390_100: SAVED_VARIANT }, + variantsById: { "1-248367227-TC-T": VARIANT }, variantTagsByGuid: { VT1726942_1248367227_r0390_100: { category: "Collaboration", color: "#668FE3", dateSaved: "2018-05-25T21:00:51.260Z", name: "Review", diff --git a/ui/shared/components/panel/variants/selectors.js b/ui/shared/components/panel/variants/selectors.js index 64afc345a5..c5583e9c59 100644 --- a/ui/shared/components/panel/variants/selectors.js +++ b/ui/shared/components/panel/variants/selectors.js @@ -16,7 +16,7 @@ import { import { getVariantTagsByGuid, getVariantNotesByGuid, getSavedVariantsByGuid, getAnalysisGroupsByGuid, getGenesById, getUser, getFamiliesByGuid, getProjectsByGuid, getIndividualsByGuid, getRnaSeqDataByIndividual, - getPhenotypeGeneScoresByIndividual, getCurrentAnalysisGroupFamilyGuids, + getPhenotypeGeneScoresByIndividual, getCurrentAnalysisGroupFamilyGuids, getVariantsById, } from 'redux/selectors' export const getIndividualGeneDataByFamilyGene = createSelector( @@ -65,8 +65,8 @@ export const getSavedVariantTableState = state => ( const matchingVariants = (variants, matchFunc) => variants.filter(o => (Array.isArray(o) ? o : [o]).some(matchFunc)) -// sorts manual variants to top of list, as manual variants are missing all populations -const sortCompHet = (a, b) => (a.populations ? 1 : 0) - (b.populations ? 1 : 0) +// sorts manual variants to top of list +const sortCompHet = (a, b) => (a.key ? 1 : 0) - (b.key ? 1 : 0) const getProjectSavedVariantsSelection = createSelector( (state, props) => props.match.params, @@ -116,7 +116,7 @@ const getSummaryDataSavedVariantsSelection = createSelector( } const pairedFilters = [] if (gene) { - pairedFilters.push(({ transcripts }) => gene in (transcripts || {})) + pairedFilters.push(({ transcripts, geneIds }) => (transcripts ? gene in transcripts : geneIds.includes(gene))) } if (tag && tag !== SHOW_ALL) { const tags = tag.split(';') pairedFilters.push(({ tagGuids }) => tags.every(t => tagGuids.some(tagGuid => ( @@ -135,25 +135,30 @@ export const getPairedSelectedSavedVariants = createSelector( getSavedVariantsByGuid, getVariantTagsByGuid, getVariantNotesByGuid, - (projectVariants, summaryDataVariants, savedVariants, tagsByGuid, notesByGuid) => { + getVariantsById, + (projectVariants, summaryDataVariants, savedVariants, tagsByGuid, notesByGuid, variantsById) => { const [variantFilter, pairedFilters] = projectVariants || summaryDataVariants - let variants = Object.values(savedVariants) + let variants = Object.values(savedVariants).map( + variant => ({ ...variant, ...(variantsById[variant.variantId] || {}) }), + ) if (variantFilter) { variants = variants.filter(variantFilter) } const selectedVariantsByGuid = variants.reduce((acc, variant) => ({ ...acc, [variant.variantGuid]: variant }), {}) - const pairedVariantsLookup = [ + const groupedVariantGuids = [ ...Object.values(tagsByGuid), ...Object.values(notesByGuid), - ].reduce((acc, { variantGuids }) => { + ].map(({ variantGuids }) => variantGuids) + const compHets = new Set(groupedVariantGuids.reduce((acc, variantGuids) => ( + variantGuids.length > 1 ? [...acc, ...variantGuids] : acc + ), [])) + + const pairedVariantsLookup = groupedVariantGuids.reduce((acc, variantGuids) => { const key = variantGuids.sort().join('_') - if (!acc[key] && variantGuids.every(variantGuid => selectedVariantsByGuid[variantGuid])) { - variantGuids.forEach((variantGuid) => { - delete acc[variantGuid] - }) + if (!acc[key] && !compHets.has(key) && variantGuids.every(variantGuid => selectedVariantsByGuid[variantGuid])) { const pairVariants = variantGuids.map(variantGuid => selectedVariantsByGuid[variantGuid]) acc[key] = pairVariants.length === 1 ? pairVariants[0] : pairVariants.sort(sortCompHet) } diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index fa5fd1b117..40a82f0f53 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -1691,8 +1691,8 @@ export const getVariantMainTranscript = ({ transcripts = {}, mainTranscriptId, s ).find(({ transcriptId }) => transcriptId === (selectedMainTranscriptId || mainTranscriptId)) || {} export const getVariantSummary = (variant, individualGuid) => { - const { alt, ref, chrom, pos, end, genomeVersion } = variant - const mainTranscript = getVariantMainTranscript(variant) + const { alt, ref, chrom, pos, end, genomeVersion, selectedMainTranscript } = variant + const mainTranscript = selectedMainTranscript || getVariantMainTranscript(variant) let consequence = `${(mainTranscript.majorConsequence || '').replace(/_variant/g, '').replace(/_/g, ' ')} variant` let variantDetail = [(mainTranscript.hgvsc || '').split(':').pop(), (mainTranscript.hgvsp || '').split(':').pop()].filter(val => val).join('/') const displayGenomeVersion = GENOME_VERSION_DISPLAY_LOOKUP[genomeVersion] || genomeVersion