diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6e138c1ca1..daf1dfe876 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -99,7 +99,7 @@ jobs: export CLICKHOUSE_READER_USER=clickhouse_read_only export CLICKHOUSE_WRITER_USER=clickhouse_test export CLICKHOUSE_SERVICE_HOSTNAME=localhost - coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data seqr matchmaker panelapp clickhouse_search + coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data clickhouse_search seqr matchmaker panelapp coverage report -m --fail-under=99 coverage report | (! grep [1-8][0-9]%) coverage report | (! grep 9[0-5]%) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/all_search_tests.py similarity index 98% rename from clickhouse_search/search_tests.py rename to clickhouse_search/all_search_tests.py index 2cf127c3f4..88f80e2dc6 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -38,7 +38,7 @@ DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, \ format_cached_variant from reference_data.models import Omim -from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults, SavedVariant, Individual +from seqr.models import Project, Family, Dataset, VariantSearch, VariantSearchResults, SavedVariant, Individual from seqr.views.apis.data_manager_api import trigger_delete_project from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, \ SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \ @@ -139,7 +139,7 @@ def assert_cached_results(self, expected_results, cache_key): self.mock_redis.expire.reset_mock() def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=None, request_body=None, check_login=None, query_params=None, search_hash=None, **search_kwargs): - search_hash = search_hash or random.randint(1000, 10000) # nosec + search_hash = search_hash or random.randint(1000, 9000000) # nosec self.mock_results_guid.return_value = f'VRS{search_hash:07d}' url = reverse(query_variants_handler, args=[search_hash]) @@ -250,8 +250,6 @@ def _get_cached_variant(cls, variant, cached_variant_fields): def _set_grch37_search(self): Project.objects.filter(id=1).update(genome_version='37') - Sample.objects.filter(sample_id='HG00732').update(is_active=False) - Sample.objects.exclude(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS).update(is_active=False) def test_single_family_search(self): variant_gene_counts = { @@ -326,7 +324,7 @@ def test_single_project_search(self): gene_counts={**variant_gene_counts, **GCNV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}} ) - self._add_sample_type_samples('WES', dataset_type='SV', guid__in=['S000135_na20870']) + self._add_sample_type_samples('WES', dataset_type='SV', guid__in=['S000129_na19675']) self._assert_expected_search( [GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], gene_counts={ 'ENSG00000129562': {'total': 1, 'families': {'F000002_2': 1, 'F000003_3': 1}}, @@ -410,17 +408,21 @@ def test_all_project_search(self): ) def test_both_sample_types_search(self): - Sample.objects.filter(dataset_type='MITO').update(is_active=False) + Dataset.objects.get(guid='S000149_hg00733').active_individuals.clear() # One family (F000011_11) in a multi-project search has identical exome and genome data. - self._add_sample_type_samples('WES', individual__family__guid='F000011_11') + self._add_sample_type_samples('WES', active_individuals__family__guid='F000011_11') self._assert_expected_search( MULTI_PROJECT_BOTH_SAMPLE_TYPE_VARIANTS, gene_counts=GENE_COUNTS, locus={'rawItems': 'chr1:1-100000000'}, project_families=MULTI_PROJECT_PROJECT_FAMILIES, check_login=self.check_collaborator_login, ) - self._add_sample_type_samples('WGS', guid__in=['S000132_hg00731']) + dataset = Dataset.objects.get(guid='S000129_na19675') + dataset.pk = None + dataset.sample_type = 'WGS' + dataset.save() + dataset.active_individuals.add(4) # Variant 1 is de novo in exome but inherited and homozygous in genome. # Variant 2 is inherited and homozygous in exome and de novo and homozygous in genome, so it fails de-novo inheritance when parental data is missing in genome. @@ -437,12 +439,13 @@ def test_both_sample_types_search(self): inheritance_mode='any_affected', quality_filter={'min_gq': 40, 'min_qs': 20}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) + self.maxDiff = None self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1], inheritance_mode='de_novo', quality_filter=None, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) - self._add_sample_type_samples('WGS', guid__in=['S000133_hg00732', 'S000134_hg00733']) + dataset.active_individuals.add(5, 6) self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], @@ -459,7 +462,6 @@ def test_both_sample_types_search(self): inheritance_mode='de_novo', quality_filter={'min_gq': 40}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) - self.maxDiff = None self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, [{**VARIANT2_BOTH_SAMPLE_TYPES, 'selectedMainTranscriptId': 'ENST00000450625'}, GCNV_VARIANT4], @@ -474,12 +476,14 @@ def test_both_sample_types_search(self): @staticmethod def _add_sample_type_samples(sample_type, dataset_type=None, **sample_filter): - for sample in Sample.objects.filter(**sample_filter): - sample.pk = None - sample.sample_type = sample_type + for dataset in Dataset.objects.filter(**sample_filter): + individuals = dataset.active_individuals.all() + dataset.pk = None + dataset.sample_type = sample_type if dataset_type: - sample.dataset_type = dataset_type - sample.save() + dataset.dataset_type = dataset_type + dataset.save() + dataset.active_individuals.set(individuals) def test_inheritance_filter(self): inheritance_mode = 'any_affected' @@ -647,10 +651,12 @@ def test_inheritance_filter(self): ) # Test deletion in trans with hom alt snp/indel - for sample in Sample.objects.filter(individual__family_id=14): - sample.pk = None - sample.dataset_type = 'SNV_INDEL' - sample.save() + dataset = Dataset.objects.get(guid='S000147_na21234') + dataset_individuals = dataset.active_individuals.all() + dataset.pk = None + dataset.dataset_type = 'SNV_INDEL' + dataset.save() + dataset.active_individuals.set(dataset_individuals) self._assert_expected_search( [[SV_VARIANT1, SV_VARIANT2], [SV_VARIANT1, PROJECT_4_COMP_HET_VARIANT], PROJECT_4_COMP_HET_VARIANT, SV_VARIANT4], inheritance_mode=inheritance_mode, inheritance_filter=sv_affected, **COMP_HET_ALL_PASS_FILTERS, @@ -678,7 +684,10 @@ def test_exclude_previous_search_results(self): request_body = {'projectFamilies': DEFAULT_PROJECT_FAMILIES, 'previousSearchHash': 'abc1234'} exclude = {'previousSearch': True} - response_search = {'exclude': {**exclude, 'previousSearchHash': 'abc1234'}} + response_search = { + 'exclude_keys': {'SNV_INDEL': [1, 2]}, + 'exclude_key_pairs': {'SNV_INDEL': [[2, 3]],'SV_WES': [[18, 19]]}, + } self._assert_expected_search( [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3], inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, @@ -692,6 +701,10 @@ def test_exclude_previous_search_results(self): self.set_cache(cache_key, [ [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3, ]) + response_search = { + 'exclude_keys': {'MITO': [8], 'SV_WES': [18]}, + 'exclude_key_pairs': {'SNV_INDEL': [[3, 4]], 'SNV_INDEL,SV_WES': [[2, 19]]}, + } self._assert_expected_search( [VARIANT2, [GCNV_VARIANT3, GCNV_VARIANT4]], exclude=exclude, **COMP_HET_ALL_PASS_FILTERS, request_body=request_body, response_search=response_search, inheritance_mode='recessive', cached_variant_fields=[ @@ -838,7 +851,7 @@ def test_location_search(self): ], ) - self._add_sample_type_samples('WES', individual__family__guid='F000014_14') + self._add_sample_type_samples('WES', active_individuals__family__guid='F000014_14') self._assert_expected_search( [SV_VARIANT1, SV_VARIANT2, MULTI_PROJECT_GCNV_VARIANT3, GCNV_VARIANT4], locus=sv_locus, project_families=[*SINGLE_FAMILY_PROJECT_FAMILIES, *SV_PROJECT_FAMILIES], @@ -858,7 +871,7 @@ def test_variant_id_search(self): [],locus={'rawVariantItems': VARIANT_IDS[1]}, ) - @mock.patch('seqr.utils.search.utils.MAX_EXPORT_VARIANTS', 2) + @mock.patch('seqr.views.apis.variant_search_api.MAX_EXPORT_VARIANTS', 2) @mock.patch('clickhouse_search.search.MAX_GENES_FOR_FILTER', 2) @mock.patch('clickhouse_search.search.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) @mock.patch('clickhouse_search.search.MAX_VARIANTS', 3) @@ -961,7 +974,7 @@ def test_invalid_search(self): self._assert_expected_search_error('Location must be specified to search across multiple projects', project_families=MULTI_PROJECT_PROJECT_FAMILIES) - Sample.objects.filter(guid='S000143_na20885').update(sample_id='HG00732') + Individual.objects.filter(guid='I000015_na20885').update(individual_id='HG00732') self._assert_expected_search_error( 'The following samples are incorrectly configured and have different affected statuses in different projects: ' 'HG00732 (1kg project nåme with uniçøde/ Test Reprocessed Project)', @@ -981,7 +994,7 @@ def test_invalid_search(self): response = self.client.get(export_url) self.assertEqual(response.status_code, 403) - @mock.patch('seqr.utils.search.utils.LiftOver') + @mock.patch('clickhouse_search.search.LiftOver') def test_variant_lookup(self, mock_liftover): mock_convert_coordinate = mock_liftover.return_value.convert_coordinate mock_convert_coordinate.side_effect = lambda chrom, pos: [(chrom, pos + 10000)] @@ -1845,7 +1858,7 @@ def test_secondary_annotations_filter(self): ], ) - self._add_sample_type_samples('WES', individual__family__guid='F000014_14') + self._add_sample_type_samples('WES', active_individuals__family__guid='F000014_14') self._assert_expected_search( [MULTI_DATA_TYPE_COMP_HET_VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], MULTI_PROJECT_GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode='recessive', @@ -1869,7 +1882,6 @@ def test_secondary_annotations_filter(self): 'familyGuids': ['F000002_2_x'], 'genotypes': {k: {**v, 'familyGuid': 'F000002_2_x'} for k, v in MULTI_DATA_TYPE_COMP_HET_VARIANT2['genotypes'].items()} } - Sample.objects.filter(guid='S000146_hg00732').update(is_active=False) Family.objects.filter(guid='F000002_2').update(guid='F000002_2_x') self._assert_expected_search( [[missing_gt_comp_het_variant, missing_gt_gcnv_variant]], @@ -2284,22 +2296,22 @@ def test_search_context(self): self.assertDictEqual( response_json['rnaSeqData']['I000001_na19675']['outliers']['ENSG00000268903'][0], {'geneId': 'ENSG00000268903', 'isSignificant': True, 'pAdjust': 1.39e-09, 'pValue': 5.88e-10, - 'tissueType': 'M', 'zScore': 7.08} + 'tissueType': 'M', 'sequencingType': 'T', 'zScore': 7.08} ) self.assertListEqual( sorted(response_json['rnaSeqData']['I000001_na19675']['spliceOutliers']['ENSG00000268903'], key=lambda d: d['start']), [{'chrom': '7', 'counts': 1297, 'end': 4000, 'geneId': 'ENSG00000268903', 'isSignificant': True, 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 0.0003, 'pValue': 0.0001, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, - 'start': 3000, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': -12.34}, + 'start': 3000, 'strand': '*', 'tissueType': 'F', 'sequencingType': 'W', 'type': 'psi5', 'deltaIntronJaccardIndex': -12.34}, {'chrom': '7', 'counts': 1297, 'end': 8000, 'geneId': 'ENSG00000268903', 'isSignificant': True, 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 0.003, 'pValue': 0.001, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, - 'start': 7000, 'strand': '*', 'tissueType': 'M', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}, + 'start': 7000, 'strand': '*', 'tissueType': 'M', 'sequencingType': 'T', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}, {'chrom': '7', 'counts': 1297, 'end': 132886973, 'geneId': 'ENSG00000268903', 'isSignificant': True, 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 3.08e-56, 'pValue': 1.08e-56, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, - 'start': 132885746, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}] + 'start': 132885746, 'strand': '*', 'tissueType': 'F', 'sequencingType': 'W', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}] ) self.assertDictEqual(response_json['savedVariantsByGuid'], {'SV0000001_2103343353_r0390_100': { **{k: mock.ANY for k in [ @@ -2392,6 +2404,7 @@ def test_search_context(self): 'category': 'CMG Discovery Tags', 'color': '#03441E', 'searchHash': None, + 'searchName': None, 'metadata': None, 'lastModifiedDate': '2018-05-29T16:32:51.449Z', 'createdBy': None, @@ -2460,9 +2473,10 @@ def test_trigger_delete_project(self): 23: {'ac_wes': 0, 'ac_wgs': 0, 'hom_wes': 0, 'hom_wgs': 0, 'ac_affected': 0, 'hom_affected': 0}, }) - project_samples = Sample.objects.filter(individual__family__project__guid='R0001_1kg', is_active=True) - self.assertEqual(project_samples.filter(dataset_type='SNV_INDEL').count(), 0) - self.assertEqual(project_samples.count(), 4) + project_datasets = Dataset.objects.filter(active_individuals__family__project__guid='R0001_1kg') + self.assertEqual(project_datasets.filter(dataset_type='SNV_INDEL').count(), 0) + self.assertEqual(project_datasets.count(), 4) + self.assertEqual(Dataset.objects.get(guid='S000129_na19675').inactive_individuals.count(), 7) body['datasetType'] = 'SV' response = self.client.post(url, content_type='application/json', data=json.dumps(body)) @@ -2473,7 +2487,7 @@ def test_trigger_delete_project(self): 'Deleted all GCNV search data for project 1kg project n\xe5me with uni\xe7\xf8de', ], }) - self.assertEqual(project_samples.filter(dataset_type='SV').count(), 0) - self.assertEqual(project_samples.count(), 1) + self.assertEqual(project_datasets.filter(dataset_type='SV').count(), 0) + self.assertEqual(project_datasets.count(), 1) diff --git a/clickhouse_search/backend/table_models.py b/clickhouse_search/backend/table_models.py index c75be39d76..c8cb45a684 100644 --- a/clickhouse_search/backend/table_models.py +++ b/clickhouse_search/backend/table_models.py @@ -3,6 +3,10 @@ from django.db.models import Func from clickhouse_search.backend.fields import NamedTupleField +from seqr.utils.logging_utils import SeqrLogger + +logger = SeqrLogger(__name__) + MATERIALIZED_VIEW_META_FIELDS = [ 'to_table', 'source_table', 'source_sql', 'source_url', 'source_url_template', 'column_selects', 'refreshable', 'create_empty', @@ -40,7 +44,8 @@ class Meta: class RefreshableMaterializedView(IncrementalMaterializedView): @classmethod - def refresh(cls): + def refresh(cls, user=None): + logger.info(f'Refreshing materialized view {cls._meta.db_table}', user) with connections['clickhouse_write'].cursor() as cursor: cursor.execute(f'SYSTEM START VIEW "{cls._meta.db_table}"') cursor.execute(f'SYSTEM REFRESH VIEW "{cls._meta.db_table}"') @@ -88,7 +93,8 @@ def dict_get_expression(cls, *expressions, field_names=None, force_tuple=False, return dict_get_func @classmethod - def reload(cls): + def reload(cls, user=None): + logger.info(f'Reloading dictionary {cls._meta.db_table}', user) with connections['clickhouse_write'].cursor() as cursor: cursor.execute(f'SYSTEM RELOAD DICTIONARY "{cls._meta.db_table}"') diff --git a/clickhouse_search/management/commands/reload_clinvar_all_variants.py b/clickhouse_search/management/commands/reload_clinvar_all_variants.py index 8f14c1a289..50c2ac478e 100644 --- a/clickhouse_search/management/commands/reload_clinvar_all_variants.py +++ b/clickhouse_search/management/commands/reload_clinvar_all_variants.py @@ -253,6 +253,40 @@ def extract_variant_info(elem: xml.etree.ElementTree.Element, new_version: str, **props, ) +def parse_clinvar_file(gzipped_file, existing_version_obj, model_to_batch, unenumerated_value_alerts): + for event, elem in ET.iterparse(gzipped_file, events=('start', 'end')): + # Handle parsing the current date. + if event == 'start' and elem.tag == 'ClinVarVariationRelease': + new_version = elem.attrib['ReleaseDate'] + if existing_version_obj: + if existing_version_obj.version == new_version: + logger.info(f'Clinvar ClickHouse tables already successfully updated to {new_version}, gracefully exiting.') + return new_version + logger.info( f'Updating Clinvar ClickHouse tables to {new_version} from {existing_version_obj and existing_version_obj.version}.') + # Drop any currently existing variants in the table that may exist due to a + # previously failed partial run. Note that we validate that the Postgresql existing version + # is present in ClickHouse to account for the situation where Postgresql has an incorrect + # version. + if existing_version_obj and ClinvarAllVariantsSnvIndel.objects.filter( + version=existing_version_obj.version).exists(): + for model in model_to_batch.keys(): + with connections['clickhouse_write'].cursor() as cursor: + cursor.execute( + f"ALTER TABLE `{model._meta.db_table}` DROP PARTITION '{new_version}';" + ) + # Handle parsing variants + if event == 'end' and elem.tag == 'VariationArchive' and new_version: + for obj in extract_variant_info(elem, new_version, unenumerated_value_alerts): + for model, batch in model_to_batch.items(): + if isinstance(obj, model): + batch.append(obj) + if len(batch) == BATCH_SIZE: + model.objects.using('clickhouse_write').bulk_create(batch) + batch.clear() + elem.clear() + + return new_version + class Command(BaseCommand): help = 'Reload all clinvar variants from weekly NCBI xml release' @@ -263,43 +297,14 @@ def handle(self, *args, **options): ClinvarAllVariantsMito: [], } unenumerated_value_alerts = [] - new_version = None + existing_version_obj = DataVersions.objects.filter(data_model_name='Clinvar').first() with tempfile.TemporaryDirectory() as tmpdir: with requests.get(WEEKLY_XML_RELEASE, stream=True, timeout=10) as r, \ tempfile.NamedTemporaryFile(dir=tmpdir, delete=False) as tmpfile: r.raise_for_status() shutil.copyfileobj(r.raw, tmpfile) with gzip.open(tmpfile.name, 'rb') as gzipped_file: - for event, elem in ET.iterparse(gzipped_file, events=('start', 'end')): - # Handle parsing the current date. - if event == 'start' and elem.tag == 'ClinVarVariationRelease': - new_version = elem.attrib['ReleaseDate'] - existing_version_obj = DataVersions.objects.filter(data_model_name='Clinvar').first() - if existing_version_obj: - if existing_version_obj.version == new_version: - logger.info(f'Clinvar ClickHouse tables already successfully updated to {new_version}, gracefully exiting.') - return - logger.info(f'Updating Clinvar ClickHouse tables to {new_version} from {existing_version_obj and existing_version_obj.version}.') - # Drop any currently existing variants in the table that may exist due to a - # previously failed partial run. Note that we validate that the Postgresql existing version - # is present in ClickHouse to account for the situation where Postgresql has an incorrect - # version. - if existing_version_obj and ClinvarAllVariantsSnvIndel.objects.filter(version=existing_version_obj.version).exists(): - for model in model_to_batch.keys(): - with connections['clickhouse_write'].cursor() as cursor: - cursor.execute( - f"ALTER TABLE `{model._meta.db_table}` DROP PARTITION '{new_version}';" - ) - # Handle parsing variants - if event == 'end' and elem.tag == 'VariationArchive' and new_version: - for obj in extract_variant_info(elem, new_version, unenumerated_value_alerts): - for model, batch in model_to_batch.items(): - if isinstance(obj, model): - batch.append(obj) - if len(batch) == BATCH_SIZE: - model.objects.using('clickhouse_write').bulk_create(batch) - batch.clear() - elem.clear() + new_version = parse_clinvar_file(gzipped_file, existing_version_obj, model_to_batch, unenumerated_value_alerts) for model, batch in model_to_batch.items(): if batch: diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py index c76769effd..0928b6d728 100644 --- a/clickhouse_search/management/commands/set_saved_variant_key.py +++ b/clickhouse_search/management/commands/set_saved_variant_key.py @@ -6,7 +6,7 @@ from clickhouse_search.search import get_clickhouse_key_lookup from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 -from seqr.models import SavedVariant, Sample +from seqr.models import SavedVariant, Dataset from seqr.utils.xpos_utils import parse_variant_id logger = logging.getLogger(__name__) @@ -22,7 +22,7 @@ def handle(self, *args, **options): saved_variant_json__populations__isnull=False, # Omit manual variants ).values_list('variant_id', flat=True).distinct() ids_by_dataset_type = { - Sample.DATASET_TYPE_VARIANT_CALLS: [], Sample.DATASET_TYPE_MITO_CALLS: [], Sample.DATASET_TYPE_SV_CALLS: [], + Dataset.DATASET_TYPE_VARIANT_CALLS: [], Dataset.DATASET_TYPE_MITO_CALLS: [], Dataset.DATASET_TYPE_SV_CALLS: [], } for variant_id in variant_ids: parsed_id = parse_variant_id(variant_id) @@ -31,23 +31,23 @@ def handle(self, *args, **options): parsed_id = parse_variant_id(variant_id[:-1]) if parsed_id: is_mito = parsed_id[0].replace('chr', '').startswith('M') - dataset_type = Sample.DATASET_TYPE_MITO_CALLS if is_mito else Sample.DATASET_TYPE_VARIANT_CALLS + dataset_type = Dataset.DATASET_TYPE_MITO_CALLS if is_mito else Dataset.DATASET_TYPE_VARIANT_CALLS else: - dataset_type = Sample.DATASET_TYPE_SV_CALLS + dataset_type = Dataset.DATASET_TYPE_SV_CALLS ids_by_dataset_type[dataset_type].append(variant_id) - no_key_mito = self._set_variant_keys(ids_by_dataset_type[Sample.DATASET_TYPE_MITO_CALLS], Sample.DATASET_TYPE_MITO_CALLS) + no_key_mito = self._set_variant_keys(ids_by_dataset_type[Dataset.DATASET_TYPE_MITO_CALLS], Dataset.DATASET_TYPE_MITO_CALLS) no_key_snv_indel = self._set_variant_keys( - ids_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] + list(no_key_mito), Sample.DATASET_TYPE_VARIANT_CALLS, + ids_by_dataset_type[Dataset.DATASET_TYPE_VARIANT_CALLS] + list(no_key_mito), Dataset.DATASET_TYPE_VARIANT_CALLS, ) if no_key_snv_indel: self._resolve_missing_variants(no_key_snv_indel, GENOME_VERSION_GRCh38) no_keys_svs = self._set_variant_keys( - ids_by_dataset_type[Sample.DATASET_TYPE_SV_CALLS], f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WGS}', + ids_by_dataset_type[Dataset.DATASET_TYPE_SV_CALLS], f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WGS}', ) - no_keys_svs = self._set_variant_keys(list(no_keys_svs), f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}') + no_keys_svs = self._set_variant_keys(list(no_keys_svs), f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WES}') if no_keys_svs: self._resolve_reloaded_svs(no_keys_svs) @@ -55,7 +55,7 @@ def handle(self, *args, **options): key__isnull=True, family__project__genome_version=GENOME_VERSION_GRCh37, ).values_list('variant_id', flat=True).distinct() - no_keys_37 = self._set_variant_keys(variant_ids_37, Sample.DATASET_TYPE_VARIANT_CALLS, genome_version=GENOME_VERSION_GRCh37) + no_keys_37 = self._set_variant_keys(variant_ids_37, Dataset.DATASET_TYPE_VARIANT_CALLS, genome_version=GENOME_VERSION_GRCh37) if no_keys_37: self._resolve_missing_variants(no_keys_37, GENOME_VERSION_GRCh37) @@ -103,7 +103,7 @@ def _query_missing_variants(cls, variant_ids, variant_fields, genome_version=GEN variant_id__in=variant_ids, family__project__genome_version=genome_version, ) num_missing = missing_variants.count() - missing_with_data_qs = missing_variants.filter(family__individual__sample__is_active=True).distinct() + missing_with_data_qs = missing_variants.filter(family__individual__active_datasets__isnull=False).distinct() missing_with_search_data = missing_with_data_qs.values( 'variant_id', *variant_fields, ).annotate(family_ids=ArrayAgg('family__family_id', distinct=True)).order_by('variant_id') @@ -128,7 +128,7 @@ def _resolve_missing_variants(cls, variant_ids, genome_version): @classmethod def _resolve_reloaded_svs(cls, variant_ids): missing_with_search_data, num_missing = cls._query_missing_variants( - list(variant_ids), ['family__individual__sample__sample_type'], + list(variant_ids), ['family__individual__active_datasets__sample_type'], ) logger.info( f'{num_missing} SV variants have no key, {num_missing - len(missing_with_search_data)} of which have no search data' @@ -139,7 +139,7 @@ def _resolve_reloaded_svs(cls, variant_ids): missing_by_sample_type = defaultdict(list) for variant in missing_with_search_data: variant_id = variant['variant_id'] - sample_type = variant['family__individual__sample__sample_type'] + sample_type = variant['family__individual__active_datasets__sample_type'] missing_by_sample_type[sample_type].append(f"{variant_id} - {'; '.join(variant['family_ids'])}" ) for sample_type, variants in missing_by_sample_type.items(): diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py index e53bdf1a2b..9b3d07f479 100644 --- a/clickhouse_search/management/tests/set_saved_variant_key_tests.py +++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py @@ -1,7 +1,7 @@ from django.core.management import call_command import mock -from seqr.models import Project, Sample, SavedVariant +from seqr.models import Project, Dataset, SavedVariant from seqr.views.utils.test_utils import AnvilAuthenticationTestCase MOCK_GCNV_DATA = [ @@ -18,7 +18,10 @@ class SetSavedVariantKeyTest(AnvilAuthenticationTestCase): @classmethod def setUpTestData(cls): Project.objects.filter(id=3).update(genome_version='38') - Sample.objects.filter(guid='S000154_na20889').update(dataset_type='SV', is_active=True) + dataset = Dataset.objects.get(guid='S000154_na20889') + dataset.dataset_type = 'SV' + dataset.save() + dataset.active_individuals.set({17}) for sv in SavedVariant.objects.filter(key__isnull=False): sv.saved_variant_json = { 'genotypes': sv.genotypes, 'populations': {'gnomad': {'af': 0.01}}, diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 7924553107..b19f8f2250 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -620,12 +620,30 @@ def _filter_annotations(self, results, *args, exclude=None, pathogenicity=None, return results + def _annotations_field_filters(self): + return { + MOTIF_FEATURES_KEY: lambda value: Q(**{ + f'sorted_{MOTIF_FEATURES_KEY}_consequences__array_exists': {'consequenceTerms': (value, 'hasAny({value}, {field})')}, + }) if hasattr(self.model, f'sorted_{MOTIF_FEATURES_KEY}_consequences') else None, + REGULATORY_FEATURES_KEY: lambda value: Q(**{ + f'sorted_{REGULATORY_FEATURES_KEY}_consequences__array_exists': {'consequenceTerms': (value, 'hasAny({value}, {field})')}, + }) if hasattr(self.model, f'sorted_{REGULATORY_FEATURES_KEY}_consequences') else None, + SCREEN_KEY: lambda value: Q(screen__in=value) if self.model.SCREEN_DICT else None, + SPLICE_AI_FIELD: lambda value: Q(preds__0__gte=float(value)) if value and SPLICE_AI_FIELD in self.entry_model.PREDICTIONS else None, + 'mitomap_pathogenic': lambda value: Q(mitomapPathogenic=value) if self.has_annotation('mitomapPathogenic') else None, + } + def _parse_annotation_filters(self, annotations, pathogenicity): filter_qs = [] allowed_consequences = [] transcript_field_filters = {} + annotations_field_filters = self._annotations_field_filters() for field, value in annotations.items(): - if field == UTR_ANNOTATOR_KEY: + if field in annotations_field_filters: + filter_q = annotations_field_filters[field](value) + if filter_q: + filter_qs.append(filter_q) + elif field == UTR_ANNOTATOR_KEY: if value: transcript_field_filters['fiveutrConsequence'] = (value, 'hasAny({value}, [{field}])') elif field == EXTENDED_SPLICE_KEY: @@ -634,21 +652,6 @@ def _parse_annotation_filters(self, annotations, pathogenicity): value = [c for c in value if c != EXTENDED_SPLICE_REGION_CONSEQUENCE] if value: allowed_consequences += value - elif field in [MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY]: - filter_field = f'sorted_{field}_consequences' - if hasattr(self.model, filter_field): - filter_qs.append(Q(**{ - f'{filter_field}__array_exists': {'consequenceTerms': (value, 'hasAny({value}, {field})')}, - })) - elif field == SCREEN_KEY: - if self.model.SCREEN_DICT: - filter_qs.append(Q(screen__in=value)) - elif field == 'mitomap_pathogenic': - if self.has_annotation('mitomapPathogenic'): - filter_qs.append(Q(mitomapPathogenic=value)) - elif field == SPLICE_AI_FIELD: - if value and SPLICE_AI_FIELD in self.entry_model.PREDICTIONS: - filter_qs.append(Q(preds__0__gte=float(value))) elif field not in SV_ANNOTATION_TYPES: allowed_consequences += value @@ -933,7 +936,7 @@ def table_basename(self): def result_values(self, *args, skip_entry_fields=True, **kwargs): return super().result_values(*args, skip_entry_fields=skip_entry_fields, **kwargs) - def join_annotations(self): + def join_annotations(self, *args, **kwargs): results = super().join_annotations(annotate_xpos=bool(self.entry_model.RANGE_PREDICTIONS)) results = results.annotate( hgmd_join=self._pathogenicity_tuple(self.variant_model.hgmd_join, 'key__hgmd_join', rename_fields={'classification': 'class'}), @@ -1050,7 +1053,7 @@ def _filter_frequency(self, entries, frequencies, **kwargs): def _join_annotations(self, entries): return self._annotate_seqr_pop_expression(entries) - def result_values(self, sample_data=None): + def result_values(self, *args, sample_data=None, **kwargs): entries = self._join_annotations(self) return self._search_call_data(entries, sample_data) diff --git a/clickhouse_search/models/__init__.py b/clickhouse_search/models/__init__.py index a34e5c1e9f..a1c506ef4a 100644 --- a/clickhouse_search/models/__init__.py +++ b/clickhouse_search/models/__init__.py @@ -13,9 +13,7 @@ state.DEFAULT_NAMES = options.DEFAULT_NAMES class ClickHouseRouter: - """ - Adapted from https://github.com/jayvynl/django-clickhouse-backend/blob/v1.3.2/README.md#configuration - """ + # Adapted from https://github.com/jayvynl/django-clickhouse-backend/blob/v1.3.2/README.md#configuration def __init__(self): self.route_model_names = set() diff --git a/clickhouse_search/models/gt_stats_models.py b/clickhouse_search/models/gt_stats_models.py index b6a12f2fb2..3076b65054 100644 --- a/clickhouse_search/models/gt_stats_models.py +++ b/clickhouse_search/models/gt_stats_models.py @@ -1,11 +1,10 @@ from clickhouse_backend import models -from django.db.models import OneToOneField, CASCADE from clickhouse_search.backend.fields import UInt32FieldDeltaCodecField, DictKeyForeignKey from clickhouse_search.backend.table_models import RefreshableMaterializedView, RefreshableMaterializedViewMeta, \ IncrementalMaterializedView, Dictionary from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 -from seqr.models import Sample +from seqr.models import Dataset class BaseProjectGtStats(models.ClickhouseModel): @@ -308,10 +307,10 @@ class Meta(GtStatsDictMeta): layout = 'FLAT(MAX_ARRAY_SIZE 5000000)' PROJECT_GT_STATS_VIEW_CLASS_MAP = { - GENOME_VERSION_GRCh37: {Sample.DATASET_TYPE_VARIANT_CALLS: ProjectsToGtStatsGRCh37SnvIndel}, + GENOME_VERSION_GRCh37: {Dataset.DATASET_TYPE_VARIANT_CALLS: ProjectsToGtStatsGRCh37SnvIndel}, GENOME_VERSION_GRCh38: { - Sample.DATASET_TYPE_VARIANT_CALLS: ProjectsToGtStatsSnvIndel, - Sample.DATASET_TYPE_MITO_CALLS: ProjectsToGtStatsMito, - Sample.DATASET_TYPE_SV_CALLS: ProjectsToGtStatsSv, + Dataset.DATASET_TYPE_VARIANT_CALLS: ProjectsToGtStatsSnvIndel, + Dataset.DATASET_TYPE_MITO_CALLS: ProjectsToGtStatsMito, + Dataset.DATASET_TYPE_SV_CALLS: ProjectsToGtStatsSv, }, } diff --git a/clickhouse_search/models/reference_data_models.py b/clickhouse_search/models/reference_data_models.py index 901fbafb61..d129f8de8b 100644 --- a/clickhouse_search/models/reference_data_models.py +++ b/clickhouse_search/models/reference_data_models.py @@ -1,5 +1,5 @@ from clickhouse_backend import models -from django.db.models import ForeignKey, OneToOneField, CASCADE, PROTECT +from django.db.models import ForeignKey, PROTECT import requests from clickhouse_search.backend.engines import Join diff --git a/clickhouse_search/models/search_models.py b/clickhouse_search/models/search_models.py index 5891057573..fb2653401c 100644 --- a/clickhouse_search/models/search_models.py +++ b/clickhouse_search/models/search_models.py @@ -10,7 +10,7 @@ from clickhouse_search.models.reference_data_models import GnomadNonCodingConstraintDict, BaseSpliceAi, \ ScreenDict from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 -from seqr.models import Sample +from seqr.models import Dataset from seqr.utils.xpos_utils import CHROMOSOME_CHOICES from settings import CLICKHOUSE_IN_MEMORY_DIR, CLICKHOUSE_DATA_DIR @@ -414,7 +414,7 @@ class Meta(BaseEntries.Meta): ) class EntriesSv(BaseEntries): - SAMPLE_TYPE = Sample.SAMPLE_TYPE_WGS + SAMPLE_TYPE = Dataset.SAMPLE_TYPE_WGS CALL_FIELDS = [ ('sampleId', models.StringField()), ('gt', models.Enum8Field(null=True, blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')])), @@ -436,7 +436,7 @@ class Meta(BaseEntries.Meta): db_table = 'GRCh38/SV/entries' class EntriesGcnv(BaseEntries): - SAMPLE_TYPE = Sample.SAMPLE_TYPE_WES + SAMPLE_TYPE = Dataset.SAMPLE_TYPE_WES CALL_FIELDS = [ ('sampleId', models.StringField()), ('gt', models.Enum8Field(null=True, blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')])), @@ -621,21 +621,21 @@ class Meta: ENTRY_CLASS_MAP = { - GENOME_VERSION_GRCh37: {Sample.DATASET_TYPE_VARIANT_CALLS: EntriesGRCh37SnvIndel}, + GENOME_VERSION_GRCh37: {Dataset.DATASET_TYPE_VARIANT_CALLS: EntriesGRCh37SnvIndel}, GENOME_VERSION_GRCh38: { - Sample.DATASET_TYPE_VARIANT_CALLS: EntriesSnvIndel, - Sample.DATASET_TYPE_MITO_CALLS: EntriesMito, - f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WGS}': EntriesSv, - f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}': EntriesGcnv, + Dataset.DATASET_TYPE_VARIANT_CALLS: EntriesSnvIndel, + Dataset.DATASET_TYPE_MITO_CALLS: EntriesMito, + f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WGS}': EntriesSv, + f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WES}': EntriesGcnv, }, } VARIANTS_CLASS_MAP = { - GENOME_VERSION_GRCh37: {Sample.DATASET_TYPE_VARIANT_CALLS: VariantsGRCh37SnvIndel}, + GENOME_VERSION_GRCh37: {Dataset.DATASET_TYPE_VARIANT_CALLS: VariantsGRCh37SnvIndel}, GENOME_VERSION_GRCh38: { - Sample.DATASET_TYPE_VARIANT_CALLS: VariantsSnvIndel, - Sample.DATASET_TYPE_MITO_CALLS: VariantsMito, - f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WGS}': VariantsSv, - f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}': VariantsGcnv, + Dataset.DATASET_TYPE_VARIANT_CALLS: VariantsSnvIndel, + Dataset.DATASET_TYPE_MITO_CALLS: VariantsMito, + f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WGS}': VariantsSv, + f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WES}': VariantsGcnv, }, } VARIANT_DETAILS_CLASS_MAP = { diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index fcd8714ee6..2c45bec171 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -7,6 +7,7 @@ from django.db.models.functions import JSONObject from django.db.utils import OperationalError import json +from pyliftover.liftover import LiftOver from clickhouse_search.backend.fields import NamedTupleField from clickhouse_search.backend.functions import Array, ArrayFilter, ArrayIntersect, ArraySort, GroupArrayArray, If, Tuple, \ @@ -16,8 +17,8 @@ from clickhouse_search.models.reference_data_models import BaseClinvar, BaseHgmd from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, EntriesSnvIndel, \ ENTRY_CLASS_MAP, VARIANTS_CLASS_MAP, VARIANT_DETAILS_CLASS_MAP -from reference_data.models import GeneInfo, GeneConstraint, Omim, GENOME_VERSION_LOOKUP, GENOME_VERSION_GRCh38 -from seqr.models import Sample, PhenotypePrioritization, Family, Individual +from reference_data.models import GeneInfo, GeneConstraint, Omim, GENOME_VERSION_LOOKUP, GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 +from seqr.models import Dataset, PhenotypePrioritization, Family, Individual from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.logging_utils import SeqrLogger from clickhouse_search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \ @@ -62,7 +63,7 @@ def get_clickhouse_variants(families, user, genome_version=None, sort=None, samp except InvalidDatasetTypeException: continue - if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and no_access_project_genome_version: + if dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS and no_access_project_genome_version: results += _get_no_access_search_results( entry_qs, variants_qs, has_comp_het, user, **search, **parsed_filters, exclude_projects=sample_data_by_dataset_type[dataset_type].get('project_guids'), inheritance_mode=inheritance_mode, @@ -92,7 +93,7 @@ def get_clickhouse_variants(families, user, genome_version=None, sort=None, samp dataset_results += _get_data_type_comp_het_results_queryset( entry_qs, variants_qs, sample_data, user, parsed_filters, **search, exclude_key_pairs=(exclude_key_pairs or {}).get(dataset_type), - is_x_chrom=has_x_chrom_comp_het and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS, + is_x_chrom=has_x_chrom_comp_het and dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS, ) if 'samples' not in sample_data: @@ -100,7 +101,7 @@ def get_clickhouse_variants(families, user, genome_version=None, sort=None, samp results += dataset_results searched_dataset_types.add(dataset_type) - if has_comp_het and any(dt.startswith(Sample.DATASET_TYPE_SV_CALLS) for dt in VARIANTS_CLASS_MAP[genome_version]): + if has_comp_het and any(dt.startswith(Dataset.DATASET_TYPE_SV_CALLS) for dt in VARIANTS_CLASS_MAP[genome_version]): results += _get_multi_data_type_comp_het_results(genome_version, families, sample_data_by_dataset_type, user, exclude_key_pairs or {}, searched_dataset_types, **search) if not searched_dataset_types: @@ -150,7 +151,6 @@ def _parse_locus_search(locus, genome_version, search): search.update({ 'genes': genes, 'intervals': intervals, 'exclude_intervals': exclude_intervals, 'raw_variant_items': locus.get('rawVariantItems'), }) - exclude.pop('rawItems', None) def get_search_genes(gene_ids, genome_version): @@ -198,8 +198,8 @@ def _raise_dataset_type_errors(sample_data_errors, sample_data_by_dataset_type): if sample_data_errors: raise InvalidSearchException(next(iter(sample_data_errors))) no_data_type = next(data_type for data_type, data in sample_data_by_dataset_type.items() if not data) - if no_data_type.startswith(Sample.DATASET_TYPE_SV_CALLS): - no_data_type = Sample.DATASET_TYPE_SV_CALLS + if no_data_type.startswith(Dataset.DATASET_TYPE_SV_CALLS): + no_data_type = Dataset.DATASET_TYPE_SV_CALLS raise InvalidSearchException(f'Unable to search against dataset type "{no_data_type}"') @@ -250,29 +250,29 @@ def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_d } try: - snv_indel_entry_qs = ENTRY_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS].objects.filter_locus(**search_kwargs) - snv_indel_variants_qs = VARIANTS_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS].objects + snv_indel_entry_qs = ENTRY_CLASS_MAP[genome_version][Dataset.DATASET_TYPE_VARIANT_CALLS].objects.filter_locus(**search_kwargs) + snv_indel_variants_qs = VARIANTS_CLASS_MAP[genome_version][Dataset.DATASET_TYPE_VARIANT_CALLS].objects snv_indel_parsed_filters = snv_indel_variants_qs.get_parsed_annotations_filters(annotations=annotations, **search_kwargs) - sv_variants_cls = VARIANTS_CLASS_MAP[genome_version][f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}'] + sv_variants_cls = VARIANTS_CLASS_MAP[genome_version][f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WES}'] sv_parsed_filters = sv_variants_cls.objects.get_parsed_annotations_filters(annotations=annotations, **search_kwargs) except InvalidDatasetTypeException: return [] - if Sample.DATASET_TYPE_VARIANT_CALLS not in sample_data_by_dataset_type: - sample_data_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] = _get_sample_data( + if Dataset.DATASET_TYPE_VARIANT_CALLS not in sample_data_by_dataset_type: + sample_data_by_dataset_type[Dataset.DATASET_TYPE_VARIANT_CALLS] = _get_sample_data( all_families, - Sample.DATASET_TYPE_VARIANT_CALLS, + Dataset.DATASET_TYPE_VARIANT_CALLS, inheritance_mode=COMPOUND_HET, inheritance_filter=search_kwargs.get('inheritance_filter'), ) - snv_indel_sample_data = sample_data_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] + snv_indel_sample_data = sample_data_by_dataset_type[Dataset.DATASET_TYPE_VARIANT_CALLS] if not (snv_indel_sample_data or {}).get('num_families'): return [] snv_indel_families = set().union(*snv_indel_sample_data['sample_type_families'].values()) results = [] - for sample_type in [Sample.SAMPLE_TYPE_WES, Sample.SAMPLE_TYPE_WGS]: - sv_dataset_type = f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' + for sample_type in [Dataset.SAMPLE_TYPE_WES, Dataset.SAMPLE_TYPE_WGS]: + sv_dataset_type = f'{Dataset.DATASET_TYPE_SV_CALLS}_{sample_type}' if sv_dataset_type not in sample_data_by_dataset_type: sample_data_by_dataset_type[sv_dataset_type] = _get_sample_data( all_families, @@ -285,7 +285,7 @@ def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_d families = snv_indel_families.intersection(sv_families) if not families: continue - logger.info(f'Loading {Sample.DATASET_TYPE_VARIANT_CALLS}/{sv_dataset_type} data for {len(families)} families', user) + logger.info(f'Loading {Dataset.DATASET_TYPE_VARIANT_CALLS}/{sv_dataset_type} data for {len(families)} families', user) snv_indel_sample_type_families = { sample_type: set(familes).intersection(sv_families) @@ -320,7 +320,7 @@ def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_d result_q = _get_comp_het_results_queryset( snv_indel_variants_qs, snv_indel_q, sv_q, len(families), - exclude_key_pairs=exclude_key_pairs.get(f'{Sample.DATASET_TYPE_VARIANT_CALLS},{sv_dataset_type}'), + exclude_key_pairs=exclude_key_pairs.get(f'{Dataset.DATASET_TYPE_VARIANT_CALLS},{sv_dataset_type}'), ) dataset_results = _evaluate_results(result_q, is_comp_het=True) if not sv_sample_data['samples']: @@ -448,9 +448,9 @@ def _add_individual_guids(results): for r in (result if isinstance(result, list) else [result]): families.update(r.get('familyGenotypes', {}).keys()) sample_map = { - (family_guid, sample_id): individual_guid for family_guid, individual_guid, sample_id in Sample.objects.filter( - individual__family__guid__in=families, is_active=True, - ).values_list('individual__family__guid', 'individual__guid', 'sample_id') + (family_guid, sample_id): individual_guid for family_guid, individual_guid, sample_id in Individual.objects.filter( + family__guid__in=families, active_datasets__isnull=False, + ).values_list('family__guid', 'guid', 'individual_id') } for result in results: if isinstance(result, list): @@ -488,7 +488,7 @@ def format_clickhouse_export_results(results): genome_version = formatted_results[0]['genomeVersion'] keys_with_no_details = {result['key'] for result in formatted_results if not 'transcripts' in result} - detail_qs = get_variant_details_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details) + detail_qs = get_variant_details_queryset(genome_version, Dataset.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details) details_by_key = { detail['key']: detail for detail in detail_qs.values( 'key', 'rsid', mainTranscript=F('transcripts__0'), variantId=F('variant_id'), @@ -523,7 +523,7 @@ def format_clickhouse_results(results): } details_by_key = { detail['key']: detail for detail in - get_variant_details_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details).result_values() + get_variant_details_queryset(genome_version, Dataset.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details).result_values() } formatted_results = [] @@ -577,66 +577,66 @@ def _is_matched_minimal_transcript(transcript, minimal_transcript): def _get_valid_samples(families, dataset_type, sample_type, allow_no_samples): - samples = Sample.objects.filter(individual__family__in=families, is_active=True) - if not samples.exists(): + individuals = Individual.objects.filter(family__in=families, active_datasets__isnull=False) + if not individuals.exists(): if allow_no_samples: return None raise InvalidSearchException(f'No search data found for families {", ".join([f.family_id for f in families])}') - samples = samples.filter(dataset_type=dataset_type) + individuals = individuals.filter(active_datasets__dataset_type=dataset_type) if sample_type: - samples = samples.filter(sample_type=sample_type) + individuals = individuals.filter(active_datasets__sample_type=sample_type) - mismatch_affected_samples = samples.values('sample_id').annotate( - projects=ArrayAgg('individual__family__project__name', distinct=True), - affected=ArrayAgg('individual__affected', distinct=True), + mismatch_affected_samples = individuals.values('individual_id').annotate( + projects=ArrayAgg('family__project__name', distinct=True), + affected=ArrayAgg('affected', distinct=True), ).filter(affected__len__gt=1) if mismatch_affected_samples: raise InvalidSearchException( 'The following samples are incorrectly configured and have different affected statuses in different projects: ' + - ', '.join([f'{agg["sample_id"]} ({"/ ".join(agg["projects"])})' for agg in mismatch_affected_samples]), + ', '.join([f'{agg["individual_id"]} ({"/ ".join(agg["projects"])})' for agg in mismatch_affected_samples]), ) - return samples + return individuals -def _get_sample_metadata(samples, affected_family_only, annotate_affected_males): - skip_individual_guid = samples.values('individual__family__project_id').distinct().count() > 1 +def _get_sample_metadata(individuals, affected_family_only, annotate_affected_males): + skip_individual_guid = individuals.values('family__project_id').distinct().count() > 1 family_array_kwargs = {'distinct': True} if affected_family_only: - family_array_kwargs['filter'] = Q(individual__affected=Individual.AFFECTED_STATUS_AFFECTED) + family_array_kwargs['filter'] = Q(affected=Individual.AFFECTED_STATUS_AFFECTED) annotations = { - 'project_guids': ArrayAgg('individual__family__project__guid', distinct=True), - 'family_guids': ArrayAgg('individual__family__guid', **family_array_kwargs), + 'project_guids': ArrayAgg('family__project__guid', distinct=True), + 'family_guids': ArrayAgg('family__guid', **family_array_kwargs), } if skip_individual_guid: annotations['num_unaffected'] = Count( - 'individual_id', distinct=True, filter=Q(individual__affected=Individual.AFFECTED_STATUS_UNAFFECTED), + 'id', distinct=True, filter=Q(affected=Individual.AFFECTED_STATUS_UNAFFECTED), ) if annotate_affected_males: - annotations['affected_male_family_guids'] = ArrayAgg('individual__family__guid', distinct=True, filter=Q( - individual__affected=Individual.AFFECTED_STATUS_AFFECTED, individual__sex__in=Individual.MALE_SEXES, + annotations['affected_male_family_guids'] = ArrayAgg('family__guid', distinct=True, filter=Q( + affected=Individual.AFFECTED_STATUS_AFFECTED, sex__in=Individual.MALE_SEXES, )) else: annotations['samples'] = ArrayAgg(JSONObject( - affected='individual__affected', sex='individual__sex', sample_id='sample_id', sample_type='sample_type', - family_guid=F('individual__family__guid'), individual_guid=F('individual__guid'), - )) + affected='affected', sex='sex', sample_id='individual_id', sample_type='active_datasets__sample_type', + family_guid=F('family__guid'), individual_guid=F('guid'), + ), distinct=True) - return samples.aggregate(**annotations) + return individuals.aggregate(**annotations) def _get_sample_data(families, dataset_type, annotate_affected_males=False, allow_no_samples=False, inheritance_mode=None, inheritance_filter=None, has_location_filter=False): sample_type = None - if dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS): + if dataset_type.startswith(Dataset.DATASET_TYPE_SV_CALLS): dataset_type, sample_type = dataset_type.split('_') - samples = _get_valid_samples(families, dataset_type, sample_type, allow_no_samples) - if not samples: + individuals = _get_valid_samples(families, dataset_type, sample_type, allow_no_samples) + if not individuals: return {} individual_affected_status = (inheritance_filter or {}).get('affected') affected_family_only = inheritance_mode and not individual_affected_status - sample_data = _get_sample_metadata(samples, affected_family_only, annotate_affected_males) + sample_data = _get_sample_metadata(individuals, affected_family_only, annotate_affected_males) family_guids = set(sample_data.pop('family_guids')) if not has_location_filter: @@ -657,8 +657,8 @@ def _get_sample_data(families, dataset_type, annotate_affected_males=False, allo sample_data['sample_type_families'][sample['sample_type']].add(sample['family_guid']) else: sample_data['sample_type_families'] = dict( - samples.filter(individual__family__guid__in=family_guids).values('sample_type').values_list( - 'sample_type', ArrayAgg('individual__family__guid', distinct=True), + individuals.filter(family__guid__in=family_guids).values('active_datasets__sample_type').values_list( + 'active_datasets__sample_type', ArrayAgg('family__guid', distinct=True), ) ) if len(sample_data['sample_type_families']) == 2: @@ -671,12 +671,12 @@ def _get_sample_data(families, dataset_type, annotate_affected_males=False, allo if set(families) - multi_families } sample_data['sample_type_families']['multi'] = multi_families - _add_missing_multi_type_samples(samples, sample_data) + _add_missing_multi_type_samples(individuals, sample_data) return sample_data -def _add_missing_multi_type_samples(samples, data): +def _add_missing_multi_type_samples(individuals, data): data['family_missing_type_samples'] = defaultdict(lambda: defaultdict(list)) if 'samples' in data: individual_sample_types = defaultdict(list) @@ -687,13 +687,13 @@ def _add_missing_multi_type_samples(samples, data): for samples in individual_sample_types.values() if len(samples) == 1 ] else: - individual_samples = samples.filter(individual__family__guid__in=data['sample_type_families']['multi'], - ).values('individual_id', family_guid=F('individual__family__guid')).annotate( - samples=ArrayAgg(JSONObject(sample_id='sample_id', sample_type='sample_type')) + individual_samples = individuals.filter(family__guid__in=data['sample_type_families']['multi'], + ).values('id', family_guid=F('family__guid')).annotate( + samples=ArrayAgg(JSONObject(sample_id='individual_id', sample_type='active_datasets__sample_type')) ).filter(samples__len=1) for agg in individual_samples: sample = agg['samples'][0] - missing_type = Sample.SAMPLE_TYPE_WES if sample['sample_type'] == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS + missing_type = Dataset.SAMPLE_TYPE_WES if sample['sample_type'] == Dataset.SAMPLE_TYPE_WGS else Dataset.SAMPLE_TYPE_WGS data['family_missing_type_samples'][agg['family_guid']][missing_type].append(sample['sample_id']) @@ -907,7 +907,7 @@ def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, aff ) except InvalidDatasetTypeException: continue - if dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS): + if dataset_type.startswith(Dataset.DATASET_TYPE_SV_CALLS): if not sample_type: raise InvalidSearchException('Sample type must be specified to look up a structural variant') elif not dataset_type.endswith(sample_type): @@ -928,9 +928,8 @@ def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, aff lifted_genome_version = next(gv for gv in ENTRY_CLASS_MAP.keys() if gv != genome_version) lifted_entry_cls = ENTRY_CLASS_MAP[lifted_genome_version].get(data_type) if lifted_entry_cls: - from seqr.utils.search.utils import run_liftover chrom, pos, base_id = variant_id.split('-', 2) - liftover_results = run_liftover(lifted_genome_version, chrom, int(pos)) + liftover_results = _run_liftover(lifted_genome_version, chrom, int(pos)) if liftover_results: lifted_id = f'{liftover_results[0]}-{liftover_results[1]}-{base_id}' entry_qs = lifted_entry_cls.objects.filter_locus(raw_variant_items=lifted_id) @@ -946,7 +945,7 @@ def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, aff if variant.get('svType') in {'DEL', 'DUP'}: other_sample_type, other_entry_class = next( (dt, cls) for dt, cls in ENTRY_CLASS_MAP[genome_version].items() - if dt != data_type and dt.startswith(Sample.DATASET_TYPE_SV_CALLS) + if dt != data_type and dt.startswith(Dataset.DATASET_TYPE_SV_CALLS) ) other_variants_cls = VARIANTS_CLASS_MAP[genome_version][other_sample_type] @@ -1001,7 +1000,7 @@ def get_variants_queryset(genome_version, dataset_type, keys, variant_ids=None): def get_variant_details_queryset(genome_version, dataset_type, keys): - if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: + if dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS: return VARIANT_DETAILS_CLASS_MAP[genome_version].objects.filter(key__in=keys) return get_variants_queryset(genome_version, dataset_type, keys) @@ -1030,12 +1029,12 @@ def _main_transcript(selected_transcript_id, sorted_transcripts): def delete_clickhouse_project(project, dataset_type, sample_type=None): - if dataset_type == Sample.DATASET_TYPE_SV_CALLS and sample_type == Sample.SAMPLE_TYPE_WES: + if dataset_type == Dataset.DATASET_TYPE_SV_CALLS and sample_type == Dataset.SAMPLE_TYPE_WES: dataset_type = 'GCNV' table_base = f'{GENOME_VERSION_LOOKUP[project.genome_version]}/{dataset_type}' partition_id = f"'{project.guid}'" partition_ids = [partition_id] - if project.genome_version == GENOME_VERSION_GRCh38 and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: + if project.genome_version == GENOME_VERSION_GRCh38 and dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS: partition_ids = [ f'({partition_id}, {pid})' for pid in EntriesSnvIndel.objects.filter(project_guid=project.guid).values_list('partition_id', flat=True).distinct() @@ -1052,3 +1051,32 @@ def delete_clickhouse_project(project, dataset_type, sample_type=None): PROJECT_GT_STATS_VIEW_CLASS_MAP[project.genome_version][dataset_type].refresh() ENTRY_CLASS_MAP[project.genome_version][dataset_type].gt_stats.rel.related_model.reload() return f'Deleted all {dataset_type} search data for project {project.name}' + + +LIFTOVERS = { + GENOME_VERSION_GRCh38: None, + GENOME_VERSION_GRCh37: None, +} +PYLIFTOVER_BUILD_LOOKUP = { + GENOME_VERSION_GRCh38: ('hg19', 'hg38'), + GENOME_VERSION_GRCh37: ('hg38', 'hg19'), +} +def _get_liftover(genome_version): + if not LIFTOVERS[genome_version]: + try: + LIFTOVERS[genome_version] = LiftOver(*PYLIFTOVER_BUILD_LOOKUP[genome_version]) + except Exception as e: + logger.error('ERROR: Unable to set up liftover. {}'.format(e), user=None) + return LIFTOVERS[genome_version] + + +def _run_liftover(genome_version, chrom, pos): + liftover = _get_liftover(genome_version) + if not liftover: + return None + lifted_coord = liftover.convert_coordinate( + 'chr{}'.format(chrom.lstrip('chr')), int(pos) + ) + if lifted_coord and lifted_coord[0]: + return (lifted_coord[0][0].lstrip('chr'), lifted_coord[0][1]) + return None diff --git a/matchmaker/views/external_api.py b/matchmaker/views/external_api.py index f6637fe456..e57e3e25e8 100644 --- a/matchmaker/views/external_api.py +++ b/matchmaker/views/external_api.py @@ -45,26 +45,12 @@ def _wrapped_view(request, *args, **kwargs): @authenticate_mme_request def mme_metrics_proxy(request, originating_node_name): - """ - -Proxies public metrics endpoint - Returns: - Metric JSON from matchbox - """ logger.info('Received MME metrics request from {}'.format(originating_node_name)) return create_json_response({'metrics': get_mme_metrics()}) @authenticate_mme_request def mme_match_proxy(request, originating_node_name): - """ - -Looks for matches for the given individual ONLY in the local MME DB. - -Expects a single patient (as per MME spec) in the POST - - Args: - None, all data in POST under key "patient_data" - Returns: - Status code and results (as per MME spec), returns raw results from MME Server - """ logger.info('Received MME match request from {}'.format(originating_node_name)) try: @@ -88,15 +74,6 @@ def mme_match_proxy(request, originating_node_name): def _safe_generate_notification_for_incoming_match(results, incoming_query, incoming_request_node, incoming_patient): - """ - Generate a SLACK notifcation to say that a VALID match request came in and the following - results were sent back. If Slack is not supported, a message is not sent, but details persisted. - - Args: - response_from_matchbox (python requests object): contains the response from matchbox - incoming_request (Django request object): The request that came into the view - incoming_patient (JSON): The query patient JSON structure from outside MME node that was matched with - """ incoming_patient_id = incoming_patient['patient']['id'] logger.info('{} MME matches found for patient {} from {}'.format( diff --git a/matchmaker/views/matchmaker_api.py b/matchmaker/views/matchmaker_api.py index 7c7abadc07..c261a5ee7f 100644 --- a/matchmaker/views/matchmaker_api.py +++ b/matchmaker/views/matchmaker_api.py @@ -35,14 +35,6 @@ @login_and_policies_required def get_individual_mme_matches(request, submission_guid): - """ - Looks for matches for the given submission. Expects a single patient (MME spec) in the POST - data field under key "patient_data" - Args: - project_id,indiv_id and POST all data in POST under key "patient_data" - Returns: - Status code and results - """ submission = MatchmakerSubmission.objects.get(guid=submission_guid) project = check_mme_permissions(submission, request.user) @@ -257,9 +249,6 @@ def _report_external_mme_error(node_name, error, detail, user, raise_exception=F @login_and_policies_required def update_mme_submission(request, submission_guid=None): - """ - Create or update the submission for the given individual. - """ submission_json = json.loads(request.body) phenotypes = submission_json.pop('phenotypes', []) gene_variants = submission_json.pop('geneVariants', []) @@ -332,9 +321,6 @@ def _get_submission_detail_response(submission, phenotypes): @login_and_policies_required def delete_mme_submission(request, submission_guid): - """ - Create or update the submission for the given individual. - """ submission = MatchmakerSubmission.objects.get(guid=submission_guid) check_mme_permissions(submission, request.user) @@ -358,14 +344,6 @@ def delete_mme_submission(request, submission_guid): @login_and_policies_required def update_mme_result_status(request, matchmaker_result_guid): - """ - Looks for matches for the given individual. Expects a single patient (MME spec) in the POST - data field under key "patient_data" - Args: - project_id,indiv_id and POST all data in POST under key "patient_data" - Returns: - Status code and results - """ result = MatchmakerResult.objects.get(guid=matchmaker_result_guid) check_mme_permissions(result.submission, request.user) @@ -380,13 +358,6 @@ def update_mme_result_status(request, matchmaker_result_guid): @login_and_policies_required def send_mme_contact_email(request, matchmaker_result_guid): - """ - Sends the given email and updates the contacted status for the match - Args: - matchmaker_result_guid - Returns: - Status code and results - """ result = MatchmakerResult.objects.get(guid=matchmaker_result_guid) check_mme_permissions(result.submission, request.user) @@ -441,14 +412,6 @@ def update_mme_project_contact(request, project_guid): @analyst_required def update_mme_contact_note(request, institution): - """ - Looks for matches for the given individual. Expects a single patient (MME spec) in the POST - data field under key "patient_data" - Args: - project_id,indiv_id and POST all data in POST under key "patient_data" - Returns: - Status code and results - """ institution = institution.strip().lower() request_json = json.loads(request.body) note, _ = get_or_create_model_from_json( @@ -509,9 +472,6 @@ def _parse_mme_result(result, hpo_terms_by_id, gene_symbols_to_ids, submission_g def _generate_notification_for_seqr_match(submission, results): - """ - Generate a notifcation to say that a match happened initiated from a seqr user. - """ matches = [] hpo_terms_by_id, genes_by_id, _ = get_mme_genes_phenotypes_for_results(results) for result in results: diff --git a/matchmaker/views/matchmaker_api_tests.py b/matchmaker/views/matchmaker_api_tests.py index 267c765fb8..2b012d6ddb 100644 --- a/matchmaker/views/matchmaker_api_tests.py +++ b/matchmaker/views/matchmaker_api_tests.py @@ -157,8 +157,8 @@ MOCK_SLACK_TOKEN = 'xoxp-123' # nosec MOCK_NODES_BY_NAME = { - 'Node A': {'name': 'Node A', 'token': 'abc', 'url': 'http://node_a.com/match'}, - 'Node B': {'name': 'Node B', 'token': 'xyz', 'url': 'http://node_b.mme.org/api'}, + 'Node A': {'name': 'Node A', 'token': 'abc', 'url': 'http://node_a.com/match'}, # nosec + 'Node B': {'name': 'Node B', 'token': 'xyz', 'url': 'http://node_b.mme.org/api'}, # nosec } class EmailException(Exception): diff --git a/panelapp/models.py b/panelapp/models.py index 9e58d53c5c..7a10f17dd3 100644 --- a/panelapp/models.py +++ b/panelapp/models.py @@ -14,8 +14,6 @@ class PaLocusList(models.Model): - """PanelApp extension of seqr.models.LocusList.""" - seqr_locus_list = models.OneToOneField( SeqrLocusList, on_delete=models.CASCADE, @@ -38,8 +36,6 @@ class Meta: class PaLocusListGene(models.Model): - """PanelApp extension of seqr.models.LocusListGene.""" - CONFIDENCE_LEVEL_CHOICES = [ ('1', 'RED'), ('2', 'AMBER'), @@ -60,8 +56,6 @@ class PaLocusListGene(models.Model): mode_of_inheritance = models.TextField(null=True, blank=True) class Meta: - """Fields included in JSON in API calls.""" - json_fields = ['confidence_level', 'mode_of_inheritance'] diff --git a/reference_data/management/tests/update_gencc_tests.py b/reference_data/management/tests/update_gencc_tests.py index 2d424b786c..af8444e88d 100644 --- a/reference_data/management/tests/update_gencc_tests.py +++ b/reference_data/management/tests/update_gencc_tests.py @@ -1,5 +1,3 @@ -import mock - from reference_data.models import GenCC from reference_data.management.tests.test_utils import ReferenceDataCommandTestCase diff --git a/reference_data/models.py b/reference_data/models.py index 91eed9fdf0..8aad05578c 100644 --- a/reference_data/models.py +++ b/reference_data/models.py @@ -31,32 +31,22 @@ class ReferenceDataRouter(object): - """ - A router to control all database operations on reference data models - """ + """A router to control all database operations on reference data models.""" + @classmethod def db_for_read(cls, model, **hints): - """ - Attempts to read reference_data models go to reference_data_db. - """ if model._meta.app_label == 'reference_data': return 'reference_data' return None @classmethod def db_for_write(cls, model, **hints): - """ - Attempts to write reference_data models go to reference_data_db. - """ if model._meta.app_label == 'reference_data': return 'reference_data' return None @classmethod def allow_relation(cls, obj1, obj2, **hints): - """ - Allow relations if a model in the reference_data app is involved. - """ if obj1._meta.app_label == 'reference_data' or \ obj2._meta.app_label == 'reference_data': return True @@ -64,10 +54,6 @@ def allow_relation(cls, obj1, obj2, **hints): @classmethod def allow_migrate(cls, db, app_label, model_name=None, **hints): - """ - Make sure the reference_data app only appears in the 'reference_data_db' - database. - """ if app_label == 'reference_data': return db == 'reference_data' elif db == 'reference_data': @@ -241,10 +227,6 @@ def _get_category_id(parent_id_map, hpo_id): return hpo_id class GeneInfo(LoadableModel): - """Human gene models from https://www.gencodegenes.org/releases/ - http://www.gencodegenes.org/gencodeformat.html - """ - ALL_GENCODE_VERSIONS = ['39', '31', '29', '28', '27', '19'] CURRENT_VERSION = ALL_GENCODE_VERSIONS[0] diff --git a/reference_data/utils/download_utils.py b/reference_data/utils/download_utils.py index 97c32148d0..c6f29b14f1 100644 --- a/reference_data/utils/download_utils.py +++ b/reference_data/utils/download_utils.py @@ -8,13 +8,6 @@ def download_file(url): - """Download the given file and returns its local path. - Args: - url (string): HTTP or FTP url - Returns: - string: local file path - """ - to_dir = tempfile.gettempdir() if not (url and url.startswith(("http://", "https://"))): raise ValueError("Invalid url: {}".format(url)) diff --git a/seqr/admin.py b/seqr/admin.py index 2ac70cc8e6..09d595c0be 100644 --- a/seqr/admin.py +++ b/seqr/admin.py @@ -1,12 +1,12 @@ from copy import deepcopy from django.contrib import admin from matchmaker.models import MatchmakerSubmission, MatchmakerIncomingQuery, MatchmakerResult, MatchmakerContactNotes -from seqr.models import Project, Family, Individual, Sample, LocusList, LocusListGene, LocusListInterval, VariantNote, \ +from seqr.models import Project, Family, Individual, Dataset, LocusList, LocusListGene, LocusListInterval, VariantNote, \ VariantTag, VariantTagType, VariantFunctionalData, SavedVariant, GeneNote, AnalysisGroup, ProjectCategory, \ FamilyAnalysedBy, VariantSearch, VariantSearchResults, IgvSample, UserPolicy, WarningMessage, FamilyNote, DynamicAnalysisGroup for model_class in [ - Project, Family, Individual, Sample, IgvSample, LocusList, LocusListGene, LocusListInterval, VariantNote, VariantTag, + Project, Family, Individual, Dataset, IgvSample, LocusList, LocusListGene, LocusListInterval, VariantNote, VariantTag, VariantTagType, VariantFunctionalData, SavedVariant, GeneNote, AnalysisGroup, ProjectCategory, FamilyAnalysedBy, VariantSearch, VariantSearchResults, MatchmakerSubmission, MatchmakerIncomingQuery, MatchmakerResult, MatchmakerContactNotes, FamilyNote, DynamicAnalysisGroup, diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index 005f532f82..4761736bdf 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -910,38 +910,33 @@ } }, { - "model": "seqr.sample", + "model": "seqr.dataset", "pk": 129, "fields": { "guid": "S000129_na19675", "created_date": "2017-02-05T06:42:55.397Z", "created_by": null, "last_modified_date": "2017-03-13T09:07:49.744Z", - - "individual": 1, + "active_individuals": [1, 4, 5, 6, 7, 9], + "inactive_individuals": [3], "sample_type": "WES", "dataset_type": "SNV_INDEL", - "sample_id": "NA19675", - "is_active": true, - "elasticsearch_index": "test_index", + "data_source": "test_index", "loaded_date": "2017-02-05T06:12:55.397Z" } }, { - "model": "seqr.sample", + "model": "seqr.dataset", "pk": 130, "fields": { "guid": "S000130_na19678", "created_date": "2017-02-05T06:42:55.397Z", "created_by": null, "last_modified_date": "2017-03-13T09:07:49.744Z", - - "individual": 2, + "active_individuals": [2], "sample_type": "WES", "dataset_type": "SNV_INDEL", - "sample_id": "NA19678", - "is_active": true, - "elasticsearch_index": "test_index_old", + "data_source": "test_index_old", "loaded_date": "2017-02-05T06:13:55.397Z" } }, @@ -956,370 +951,97 @@ "individual": 19, "is_active": true, "tissue_type": "F", + "sequencing_type": "W", "data_source": "fibs_samples.tsv.gz", "data_type": "S" } }, { - "model": "seqr.sample", - "pk": 131, - "fields": { - "guid": "S000131_na19679", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:49.800Z", - - "sample_id": "NA19679", - "sample_type": "WES", - "is_active": false, - "elasticsearch_index": "test_index", - "individual": 3, - "dataset_type": "SNV_INDEL", - "loaded_date": "2017-02-05T06:15:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 132, - "fields": { - "guid": "S000132_hg00731", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:49.937Z", - "elasticsearch_index": "test_index", - "sample_id": "HG00731", - "sample_type": "WES", - "is_active": true, - "individual": 4, - "dataset_type": "SNV_INDEL", - "loaded_date": "2017-02-05T06:16:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 133, - "fields": { - "guid": "S000133_hg00732", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:49.963Z", - "elasticsearch_index": "test_index", - "sample_id": "HG00732", - "sample_type": "WES", - "is_active": true, - "individual": 5, - "dataset_type": "SNV_INDEL", - "loaded_date": "2017-02-05T06:17:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 134, - "fields": { - "guid": "S000134_hg00733", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:49.990Z", - "elasticsearch_index": "test_index", - "sample_id": "HG00733", - "sample_type": "WES", - "is_active": true, - "individual": 6, - "dataset_type": "SNV_INDEL", - "loaded_date": "2017-02-05T06:18:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 135, - "fields": { - "guid": "S000135_na20870", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:50.022Z", - "elasticsearch_index": "test_index", - "sample_id": "NA20870", - "sample_type": "WES", - "is_active": true, - "individual": 7, - "dataset_type": "SNV_INDEL", - "loaded_date": "2017-02-05T06:19:55.397Z" - } -}, -{ - "model": "seqr.sample", + "model": "seqr.dataset", "pk": 136, "fields": { "guid": "S000136_na20872", "created_date": "2017-02-05T06:42:55.397Z", "created_by": null, "last_modified_date": "2017-03-13T09:07:50.052Z", - "sample_id": "NA20872", "sample_type": "WES", - "is_active": false, - "individual": 8, + "inactive_individuals": [8, 10, 11, 12, 14], "dataset_type": "SNV_INDEL", - "elasticsearch_index": "1kg.vcf.gz", + "data_source": "1kg.vcf.gz", "loaded_date": "2017-02-05T06:20:55.397Z" } }, { - "model": "seqr.sample", - "pk": 137, - "fields": { - "guid": "S000137_na20874", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:50.079Z", - "elasticsearch_index": "test_index", - "sample_id": "NA20874", - "sample_type": "WES", - "is_active": true, - "individual": 9, - "dataset_type": "SNV_INDEL", - "loaded_date": "2017-02-05T06:21:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 138, - "fields": { - "guid": "S000138_na20875", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:50.111Z", - "sample_id": "NA20875", - "sample_type": "WES", - "is_active": false, - "individual": 10, - "dataset_type": "SNV_INDEL", - "elasticsearch_index": "1kg.vcf.gz", - "loaded_date": "2017-02-05T06:22:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 139, - "fields": { - "guid": "S000139_na20876", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:50.136Z", - - "sample_id": "NA20876", - "sample_type": "WES", - "is_active": false, - "individual": 11, - "dataset_type": "SNV_INDEL", - "elasticsearch_index": "1kg.vcf.gz", - "loaded_date": "2017-02-05T06:23:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 140, - "fields": { - "guid": "S000140_na20877", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:50.165Z", - - "sample_id": "NA19678", - "sample_type": "WES", - "is_active": false, - "individual": 12, - "dataset_type": "SNV_INDEL", - "elasticsearch_index": "1kg.vcf.gz", - "loaded_date": "2017-02-05T06:24:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 142, - "fields": { - "guid": "S000142_na20881", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:50.220Z", - - "sample_id": "NA20881", - "sample_type": "WES", - "is_active": false, - "individual": 14, - "dataset_type": "SNV_INDEL", - "elasticsearch_index": "1kg.vcf.gz", - "loaded_date": "2017-02-05T06:25:55.397Z" - } -}, -{ - "model": "seqr.sample", + "model": "seqr.dataset", "pk": 143, "fields": { "guid": "S000143_na20885", "created_date": "2020-02-05T06:42:55.397Z", "created_by": null, "last_modified_date": "2020-03-13T09:07:50.247Z", - "elasticsearch_index": "test_index_second", - "sample_id": "NA20885", + "data_source": "test_index_second", "sample_type": "WGS", - "is_active": true, - "individual": 15, + "active_individuals": [15], "dataset_type": "SNV_INDEL", "loaded_date": "2020-02-05T06:26:55.397Z" } }, { - "model": "seqr.sample", + "model": "seqr.dataset", "pk": 144, "fields": { "guid": "S000144_na20888", "created_date": "2017-02-05T06:42:55.397Z", "created_by": null, "last_modified_date": "2017-03-13T09:07:50.277Z", - - "sample_id": "NA20888", "sample_type": "WGS", - "is_active": false, - "individual": 16, + "inactive_individuals": [16], "dataset_type": "SNV_INDEL", - "elasticsearch_index": "1kg.vcf.gz", + "data_source": "1kg.vcf.gz", "loaded_date": "2017-02-05T06:27:55.397Z" } }, { - "model": "seqr.sample", - "pk": 154, - "fields": { - "guid": "S000154_na20889", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:50.277Z", - - "sample_id": "NA20889", - "sample_type": "WES", - "is_active": false, - "individual": 17, - "dataset_type": "SNV_INDEL", - "elasticsearch_index": "1kg.vcf.gz", - "data_source": "auto__2023-08-08", - "loaded_date": "2017-02-05T06:28:55.397Z" - } -}, -{ - "model": "seqr.sample", + "model": "seqr.dataset", "pk": 145, "fields": { "guid": "S000145_hg00731", "created_date": "2018-02-05T06:42:55.397Z", "created_by": null, "last_modified_date": "2018-03-13T09:07:49.937Z", - "elasticsearch_index": "test_index_sv", - "sample_id": "HG00731", + "data_source": "test_index_sv", "sample_type": "WES", - "is_active": true, - "individual": 4, + "active_individuals": [4, 5, 6], "dataset_type": "SV", "loaded_date": "2018-02-05T06:29:55.397Z" } }, { - "model": "seqr.sample", - "pk": 146, - "fields": { - "guid": "S000146_hg00732", - "created_date": "2018-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2018-03-13T09:07:49.963Z", - "elasticsearch_index": "test_index_sv", - "sample_id": "HG00732", - "sample_type": "WES", - "is_active": true, - "individual": 5, - "dataset_type": "SV", - "loaded_date": "2018-02-05T06:30:55.397Z" - } -}, -{ - "model": "seqr.sample", + "model": "seqr.dataset", "pk": 147, "fields": { "guid": "S000147_na21234", "created_date": "2018-02-05T06:42:55.397Z", "created_by": null, "last_modified_date": "2018-03-13T09:07:49.937Z", - "elasticsearch_index": "test_index_sv_wgs", - "sample_id": "NA21234", + "data_source": "test_index_sv_wgs", "sample_type": "WGS", - "is_active": true, - "individual": 18, + "active_individuals": [18, 19, 21], "dataset_type": "SV", "loaded_date": "2018-02-05T06:31:55.397Z" } }, { - "model": "seqr.sample", - "pk": 173, - "fields": { - "guid": "S000173_na21987", - "created_date": "2018-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2018-03-13T09:07:49.937Z", - "elasticsearch_index": "test_index_sv_wgs", - "sample_id": "NA21987", - "sample_type": "WGS", - "is_active": true, - "individual": 19, - "dataset_type": "SV", - "loaded_date": "2018-02-05T06:31:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 174, - "fields": { - "guid": "S000174_na21654", - "created_date": "2018-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2018-03-13T09:07:49.937Z", - "elasticsearch_index": "test_index_sv_wgs", - "sample_id": "NA21654", - "sample_type": "WGS", - "is_active": true, - "individual": 21, - "dataset_type": "SV", - "loaded_date": "2018-02-05T06:31:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 148, - "fields": { - "guid": "S000148_hg00733", - "created_date": "2018-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2018-03-13T09:07:49.963Z", - "elasticsearch_index": "test_index_sv", - "sample_id": "HG00733", - "sample_type": "WES", - "is_active": true, - "individual": 6, - "dataset_type": "SV", - "loaded_date": "2018-02-05T06:32:55.397Z" - } -}, -{ - "model": "seqr.sample", + "model": "seqr.dataset", "pk": 149, "fields": { "guid": "S000149_hg00733", "created_date": "2022-02-05T06:42:55.397Z", "created_by": null, "last_modified_date": "2022-03-13T09:07:49.937Z", - "elasticsearch_index": "test_index_mito_wgs", - "sample_id": "HG00731", + "data_source": "test_index_mito_wgs", "sample_type": "WES", - "is_active": true, - "individual": 4, + "active_individuals": [4], "dataset_type": "MITO", "loaded_date": "2022-02-05T06:33:55.397Z" } @@ -1335,6 +1057,7 @@ "individual": 1, "is_active": true, "tissue_type": "F", + "sequencing_type": "W", "data_source": "muscle_samples.tsv.gz", "data_type": "S" } @@ -1350,10 +1073,26 @@ "individual": 1, "is_active": true, "tissue_type": "M", + "sequencing_type": "T", "data_source": "muscle_samples.tsv.gz", "data_type": "S" } }, +{ + "model": "seqr.dataset", + "pk": 154, + "fields": { + "guid": "S000154_na20889", + "created_date": "2017-02-05T06:42:55.397Z", + "created_by": null, + "last_modified_date": "2017-03-13T09:07:50.277Z", + "sample_type": "WES", + "inactive_individuals": [17], + "dataset_type": "SNV_INDEL", + "data_source": "auto__2023-08-08", + "loaded_date": "2017-02-05T06:28:55.397Z" + } +}, { "model": "seqr.rnasample", "pk": 161, @@ -1365,6 +1104,7 @@ "individual": 1, "is_active": true, "tissue_type": "F", + "sequencing_type": "W", "data_source": "muscle_samples.tsv.gz", "data_type": "T" } @@ -1380,6 +1120,7 @@ "individual": 1, "is_active": true, "tissue_type": "M", + "sequencing_type": "T", "data_source": "muscle_samples.tsv.gz", "data_type": "T" } @@ -1395,6 +1136,7 @@ "individual": 1, "is_active": true, "tissue_type": "M", + "sequencing_type": "T", "data_source": "muscle_samples.tsv.gz", "data_type": "E" } @@ -2204,6 +1946,24 @@ "search_hash": "125897d6" } }, +{ + "model": "seqr.variantsearchresults", + "pk": 123, + "fields": { + "guid": "VSR000123_125897d6", + "variant_search_id": 1234, + "search_hash": "125897d6" + } +}, + { + "model": "seqr.variantsearch", + "pk": 1234, + "fields": { + "guid": "VS0001234_incomplete_penetranc", + "name": "Incomplete Penetrance De-novo", + "search": {} + } +}, { "model": "seqr.varianttag", "pk": 1726945, diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 3ce9618d97..444473d486 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -10,12 +10,13 @@ from clickhouse_search.search import get_clickhouse_genotypes from reference_data.models import GENOME_VERSION_LOOKUP -from seqr.models import Family, Sample, Project, Individual, SavedVariant +from seqr.models import Family, Dataset, Project, Individual, SavedVariant from seqr.utils.communication_utils import safe_post_to_slack, send_project_email from seqr.utils.file_utils import file_iter, list_files, is_google_bucket_file_path -from seqr.utils.search.add_data_utils import notify_search_data_loaded, update_airtable_loading_tracking_status +from seqr.utils.add_data_utils import notify_search_data_loaded, update_airtable_loading_tracking_status from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS from seqr.views.utils.export_utils import write_multiple_files +from seqr.views.utils.json_to_orm_utils import create_model_from_json from seqr.views.utils.permissions_utils import is_internal_anvil_project, project_has_anvil from seqr.views.utils.variant_utils import reset_cached_search_results from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, PIPELINE_DATA_DIR, ANVIL_UI_URL, IS_ANVIL_LOADING_DELAY, \ @@ -30,10 +31,10 @@ ERRORS_REPORTED_FILE_NAME = '_ERRORS_REPORTED' RUN_PATH_FIELDS = ['genome_version', 'dataset_type', 'run_version', 'file_name'] -DATASET_TYPE_MAP = {'GCNV': Sample.DATASET_TYPE_SV_CALLS} +DATASET_TYPE_MAP = {'GCNV': Dataset.DATASET_TYPE_SV_CALLS} CLICKHOUSE_DATASET_TYPE_MAP = { - 'GCNV': f'{Sample.DATASET_TYPE_SV_CALLS}_WES', - Sample.DATASET_TYPE_SV_CALLS: f'{Sample.DATASET_TYPE_SV_CALLS}_WGS', + 'GCNV': f'{Dataset.DATASET_TYPE_SV_CALLS}_WES', + Dataset.DATASET_TYPE_SV_CALLS: f'{Dataset.DATASET_TYPE_SV_CALLS}_WGS', } RELATEDNESS_CHECK_NAME = 'relatedness_check' @@ -76,7 +77,7 @@ def handle(self, *args, **options): def _load_success_runs(self, runs, success_run_dirs): - loaded_runs = set(Sample.objects.filter(data_source__isnull=False).values_list('data_source', flat=True)) + loaded_runs = {source for sources in Dataset.objects.values_list('data_source', flat=True) for source in sources.split(',')} new_runs = { run_dir: run_details for run_dir, run_details in runs.items() if run_dir in success_run_dirs and run_details['run_version'] not in loaded_runs @@ -206,18 +207,29 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers family_project_map = {f.guid: f.project for f in families.select_related('project')} families_by_project = defaultdict(list) samples_by_project = defaultdict(list) + num_samples = 0 for family_guid, sample_ids in metadata['family_samples'].items(): project = family_project_map[family_guid] families_by_project[project].append(family_guid) samples_by_project[project] += sample_ids + num_samples += len(sample_ids) - sample_project_tuples = [] + individuals_by_project = {} + missing_samples = set() + all_individual_ids = set() invalid_genome_version_projects = [] for project, sample_ids in samples_by_project.items(): - sample_project_tuples += [(sample_id, project.id) for sample_id in sample_ids] project_genome_version = GENOME_VERSION_LOOKUP.get(project.genome_version, project.genome_version) if project_genome_version != genome_version: invalid_genome_version_projects.append((project.guid, project_genome_version)) + continue + + matched_individuals = dict( + Individual.objects.filter(family__project=project, individual_id__in=sample_ids).values_list('id', 'individual_id') + ) + missing_samples.update(set(sample_ids) - set(matched_individuals.values())) + individuals_by_project[project] = matched_individuals + all_individual_ids.update(matched_individuals.keys()) if invalid_genome_version_projects: raise CommandError( @@ -226,14 +238,18 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers ) sample_type = metadata['sample_type'] - logger.info(f'Loading {len(sample_project_tuples)} {sample_type} {dataset_type} samples in {len(samples_by_project)} projects') - new_samples = cls._match_and_update_search_samples( - sample_project_tuples, sample_type, dataset_type, data_source=run_version, elasticsearch_index=';'.join(metadata['callsets']), - ) + logger.info(f'Loading {num_samples} {sample_type} {dataset_type} samples in {len(samples_by_project)} projects') + if missing_samples: + sample_ids = ', '.join(sorted(missing_samples)) + raise ValueError(f'Matches not found for sample ids: {sample_ids}') - new_samples_by_project = dict(new_samples.values('individual__family__project').annotate( - samples=ArrayAgg('sample_id', distinct=True), - ).values_list('individual__family__project', 'samples')) + cls._update_matched_families(all_individual_ids, dataset_type, sample_type) + + new_samples_by_project = {} + for project, individuals in individuals_by_project.items(): + new_samples_by_project[project.id] = cls._match_and_update_search_datasets( + individuals, sample_type, dataset_type, data_source=run_version, + ) split_project_pdos = cls._report_loading_success( dataset_type, sample_type, run_version, samples_by_project, new_samples_by_project, @@ -271,7 +287,7 @@ def _report_loading_success(cls, dataset_type, sample_type, run_version, samples project, is_internal, dataset_type, sample_type, new_samples_by_project.get(project.id, []), num_samples=len(sample_ids), ) - if session and is_internal and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: + if session and is_internal and dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS: split_project_pdos[project.name] = cls._update_pdos(session, project.guid, sample_ids) except Exception as e: logger.error(f'Error reporting loading success for project {project.name} in {run_version}: {e}') @@ -412,72 +428,38 @@ def _update_project_saved_variant_genotypes(project, family_guids, dataset_type) return updates @classmethod - def _match_and_update_search_samples(cls, sample_project_tuples, sample_type, dataset_type, **sample_data): - individual_ids_by_keys = cls._get_matched_individuals(sample_project_tuples) - individual_ids = individual_ids_by_keys.values() - matched_samples = Sample.objects.filter(is_active=True, dataset_type=dataset_type, sample_type=sample_type) - cls._update_matched_families(individual_ids, matched_samples) - - sample_guids_by_individual = dict(Sample.objects.filter( - individual_id__in=individual_ids, sample_type=sample_type, dataset_type=dataset_type, **sample_data, - ).values_list('individual_id', 'guid')) - remaining_key_individuals = { - key: individual_id for key, individual_id in individual_ids_by_keys.items() - if individual_id not in sample_guids_by_individual - } + def _match_and_update_search_datasets(cls, individuals, sample_type, dataset_type, data_source): loaded_date = timezone.now() - created_sample_guids = cls._create_samples( - remaining_key_individuals, loaded_date=loaded_date, sample_type=sample_type, dataset_type=dataset_type, - **sample_data, - ) if remaining_key_individuals else [] - - sample_guids = list(sample_guids_by_individual.values()) + created_sample_guids - - activated_sample_guids = Sample.bulk_update(user=None, update_json={ - 'is_active': True, - 'loaded_date': loaded_date, - **sample_data, - }, guid__in=sample_guids, is_active=False) - - inactivate_samples = matched_samples.filter(individual_id__in=individual_ids).exclude(guid__in=sample_guids) - inactivated_sample_guids = Sample.bulk_update( - user=None, update_json={'is_active': False}, queryset=inactivate_samples, - ) - - previous_loaded_individuals = set(Sample.objects.filter(guid__in=inactivated_sample_guids).values_list('individual_id', flat=True)) - return Sample.objects.filter(guid__in=activated_sample_guids).exclude(individual_id__in=previous_loaded_individuals) + dataset = create_model_from_json(Dataset, { + 'dataset_type': dataset_type, 'sample_type': sample_type, 'data_source': data_source, 'loaded_date': loaded_date, + }, user=None) + dataset.active_individuals.set(individuals.keys()) + + inactivate_datasets = Dataset.objects.filter( + dataset_type=dataset_type, sample_type=sample_type, active_individuals__id__in=individuals.keys(), + ).exclude(id=dataset.id) + loaded_individuals = set() + for dataset in inactivate_datasets: + inactivate_individuals = dataset.active_individuals.filter(id__in=individuals.keys()) + loaded_individuals.update(inactivate_individuals.values_list('id', flat=True)) + dataset.inactive_individuals.add(*inactivate_individuals) + dataset.active_individuals.remove(*inactivate_individuals) + + return [sample_id for individual_id, sample_id in individuals.items() if individual_id not in loaded_individuals] @staticmethod - def _get_matched_individuals(sample_project_tuples): - individual_ids_by_keys = { - (individual_id, project_id): individual_db_id - for individual_db_id, individual_id, project_id in Individual.objects.filter( - family__project_id__in={project_id for _, project_id in sample_project_tuples}, - individual_id__in={sample_id for sample_id, _ in sample_project_tuples}, - ).values_list('id', 'individual_id', 'family__project_id') - if (individual_id, project_id) in sample_project_tuples - } - - missing_keys = set(sample_project_tuples) - set(individual_ids_by_keys.keys()) - if missing_keys: - sample_ids = ', '.join(sorted([sample_id for sample_id, _ in missing_keys])) - raise ValueError(f'Matches not found for sample ids: {sample_ids}') - - return individual_ids_by_keys - - @staticmethod - def _update_matched_families(individual_ids, matched_samples): + def _update_matched_families(individual_ids, dataset_type, sample_type): included_families = dict( Family.objects.filter(individual__id__in=individual_ids).values_list('id', 'analysis_status') ) - missing_individuals = matched_samples.filter( - individual__family_id__in=included_families, - ).exclude(individual_id__in=individual_ids).values( - 'individual__family__family_id', - ).annotate(individual_ids=ArrayAgg('individual__individual_id', ordering='individual__individual_id')) + missing_individuals = Individual.objects.filter( + family_id__in=included_families, active_datasets__dataset_type=dataset_type, active_datasets__sample_type=sample_type, + ).exclude(id__in=individual_ids).values( + 'family__family_id', + ).annotate(individual_ids=ArrayAgg('individual_id', ordering='individual_id')) if missing_individuals: missing_summary = ', '.join(sorted([ - f"{agg['individual__family__family_id']} ({', '.join(agg['individual_ids'])})" for agg in missing_individuals + f"{agg['family__family_id']} ({', '.join(agg['individual_ids'])})" for agg in missing_individuals ])) raise ValueError( f'The following families are included in the callset but are missing some family members: {missing_summary}' @@ -491,17 +473,6 @@ def _update_matched_families(individual_ids, matched_samples): user=None, update_json={'analysis_status': Family.ANALYSIS_STATUS_ANALYSIS_IN_PROGRESS}, id__in=family_ids_to_update, ) - @staticmethod - def _create_samples(remaining_key_individuals, **sample_params): - new_samples = [ - Sample( - individual_id=individual_id, - sample_id=sample_key[0], - **sample_params, - ) for sample_key, individual_id in remaining_key_individuals.items()] - new_sample_models = Sample.bulk_create(user=None, new_models=new_samples) - return [s.guid for s in new_sample_models] - update_individuals_sample_qc = Command._update_individuals_sample_qc get_pipeline_runs = Command._get_runs diff --git a/seqr/management/commands/reload_saved_variant_genotypes.py b/seqr/management/commands/reload_saved_variant_genotypes.py index 3cd146ac40..0af32763aa 100644 --- a/seqr/management/commands/reload_saved_variant_genotypes.py +++ b/seqr/management/commands/reload_saved_variant_genotypes.py @@ -1,7 +1,7 @@ import logging from django.core.management.base import BaseCommand from seqr.management.commands.check_for_new_samples_from_pipeline import update_project_saved_variant_genotypes -from seqr.models import Project, Family, Sample +from seqr.models import Project, Family, Dataset logger = logging.getLogger(__name__) @@ -18,12 +18,12 @@ def handle(self, *args, **options): family_guid = options['family_guid'] family_guids = [family_guid] if family_guid else Family.objects.filter(project=project).values_list('guid', flat=True) - samples = Sample.objects.filter(individual__family__project_id=project.id, is_active=True) + datasets = Dataset.objects.filter(active_individuals__family__project_id=project.id) if family_guid: - samples = samples.filter(individual__family__guid=family_guid) + datasets = datasets.filter(active_individuals__family__guid=family_guid) dataset_types = { - f'{dataset_type}_{sample_type}' if dataset_type == Sample.DATASET_TYPE_SV_CALLS else dataset_type - for dataset_type, sample_type in samples.values_list('dataset_type', 'sample_type').distinct() + f'{dataset_type}_{sample_type}' if dataset_type == Dataset.DATASET_TYPE_SV_CALLS else dataset_type + for dataset_type, sample_type in datasets.values_list('dataset_type', 'sample_type').distinct() } for dataset_type in sorted(dataset_types): update_project_saved_variant_genotypes(project, family_guids, dataset_type) diff --git a/seqr/management/commands/reset_cached_search_results.py b/seqr/management/commands/reset_cached_search_results.py index 5adcbb33c4..8b75e72366 100644 --- a/seqr/management/commands/reset_cached_search_results.py +++ b/seqr/management/commands/reset_cached_search_results.py @@ -14,7 +14,6 @@ def add_arguments(self, parser): parser.add_argument('--reset-index-metadata', action='store_true', help='reset the index metadata') def handle(self, *args, **options): - """transfer project""" project_name = options['project'] project = Project.objects.get(name=project_name) if project_name else None reset_cached_search_results(project=project, reset_index_metadata=options.get('reset_index_metadata')) diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index c59138e3f2..a7807a4077 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -9,7 +9,7 @@ from clickhouse_search.search import get_clickhouse_variants, get_search_genes, ENTRY_CLASS_MAP from panelapp.models import PaLocusListGene from reference_data.models import GENOME_VERSION_GRCh38 -from seqr.models import Project, Family, Individual, Sample, LocusList +from seqr.models import Project, Family, Individual, Dataset, LocusList from seqr.utils.communication_utils import send_project_notification from clickhouse_search.constants import ANY_AFFECTED, HOMOZYGOUS_RECESSIVE, X_LINKED_RECESSIVE_MALE_AFFECTED, DE_NOVO, COMPOUND_HET from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets @@ -482,7 +482,7 @@ def handle(self, *args, **options): updates = {update: set() for update in ['matched_families', 'new_tag_keys', 'update_tag_keys', 'skipped_tag_keys']} search_counts = {} samples_by_dataset_type = {} - sample_qs = Sample.objects.filter(individual__family__project=project, is_active=True) + sample_qs = Individual.objects.filter(family__project=project) for dataset_type, searches in SEARCHES.items(): self._run_dataset_type_searches( dataset_type, searches, sample_qs, updates, search_counts, samples_by_dataset_type, family_guid_map, @@ -501,7 +501,7 @@ def handle(self, *args, **options): return family_new_counts = defaultdict(int) - for family_id, variant_id in new_tag_keys: + for family_id, _ in new_tag_keys: family_new_counts[family_id] += 1 send_project_notification( @@ -517,25 +517,25 @@ def handle(self, *args, **options): @classmethod def _run_dataset_type_searches(cls, dataset_type, searches, sample_qs, updates, search_counts, samples_by_dataset_type, family_guid_map, project, exclude_intervals, gene_by_moi): - is_sv = dataset_type == Sample.DATASET_TYPE_SV_CALLS - sample_qs = sample_qs.filter(dataset_type=dataset_type) + is_sv = dataset_type == Dataset.DATASET_TYPE_SV_CALLS + sample_qs = sample_qs.filter(active_datasets__dataset_type=dataset_type) if is_sv: sample_qs = sample_qs.exclude( - individual__sv_flags__contains=['outlier_num._calls'], individual__affected=Individual.AFFECTED_STATUS_AFFECTED, + sv_flags__contains=['outlier_num._calls'], affected=Individual.AFFECTED_STATUS_AFFECTED, ) - sample_types = list(sample_qs.values_list('sample_type', flat=True).distinct()) + sample_types = list(sample_qs.values_list('active_datasets__sample_type', flat=True).distinct()) if len(sample_types) > 1: raise CommandError('Variant prioritization not supported for projects with multiple sample types') sample_type = sample_types[0] if is_sv: dataset_type = f'{dataset_type}_{sample_type}' samples_by_family = { - agg['individual__family__guid']: agg for agg in sample_qs.values('individual__family__guid').annotate( + agg['family__guid']: agg for agg in sample_qs.values('family__guid').annotate( affecteds=ArrayAgg( - JSONObject(maternal_guid='individual__mother__guid', paternal_guid='individual__father__guid', sex='individual__sex'), - filter=Q(individual__affected=Individual.AFFECTED_STATUS_AFFECTED), + JSONObject(maternal_guid='mother__guid', paternal_guid='father__guid', sex='sex'), + filter=Q(affected=Individual.AFFECTED_STATUS_AFFECTED), ), - unaffected_guids=ArrayAgg('individual__guid', filter=Q(individual__affected=Individual.AFFECTED_STATUS_UNAFFECTED)), + unaffected_guids=ArrayAgg('guid', filter=Q(affected=Individual.AFFECTED_STATUS_UNAFFECTED)), ).filter(affecteds__len__gt=0) } samples_by_dataset_type[dataset_type] = samples_by_family @@ -608,12 +608,12 @@ def wrapped(new_variant_keys, family_variant_data): def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by_dataset_type, family_guid_map, project, genes): sv_dataset_type = next(dt for dt in samples_by_dataset_type.keys() if dt.startswith('SV')) sample_type = sv_dataset_type.split('_')[-1] - families = set(samples_by_dataset_type[sv_dataset_type].keys()).intersection(samples_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS].keys()) + families = set(samples_by_dataset_type[sv_dataset_type].keys()).intersection(samples_by_dataset_type[Dataset.DATASET_TYPE_VARIANT_CALLS].keys()) sv_samples_by_family = { guid: sample_data for guid, sample_data in samples_by_dataset_type[sv_dataset_type].items() if guid in families } snv_indel_samples_by_family = { - guid: sample_data for guid, sample_data in samples_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS].items() + guid: sample_data for guid, sample_data in samples_by_dataset_type[Dataset.DATASET_TYPE_VARIANT_CALLS].items() if guid in families } family_variant_data = defaultdict(lambda: {'matched_searches': set(), 'matched_comp_het_searches': set(), 'support_vars': set()}) @@ -627,7 +627,7 @@ def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by project, sample_type, snv_indel_samples_by_family, config_search.get('family_filter'), ) sample_data_by_dataset_type = { - Sample.DATASET_TYPE_VARIANT_CALLS: snv_indel_sample_data, sv_dataset_type: sv_sample_data, + Dataset.DATASET_TYPE_VARIANT_CALLS: snv_indel_sample_data, sv_dataset_type: sv_sample_data, } num_results = cls._execute_search( sample_data_by_dataset_type, search_name, family_variant_data, family_guid_map, diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index 356a0a615c..5c5e401b73 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -1,7 +1,8 @@ from django.core.management.base import BaseCommand +from django.db.models import Q -from seqr.models import Project, Family, VariantTag, VariantTagType -from seqr.utils.search.add_data_utils import trigger_delete_families_search +from seqr.models import Project, Family, VariantTag, VariantTagType, Dataset +from seqr.utils.add_data_utils import trigger_delete_families_search import logging logger = logging.getLogger(__name__) @@ -36,6 +37,19 @@ def handle(self, *args, **options): trigger_delete_families_search(from_project, list(families.values_list('guid', flat=True))) + remaining_families = Family.objects.filter(project=from_project).exclude(id__in={f.id for f in families}) + split_dataset = Dataset.objects.filter(inactive_individuals__family__in=families).filter( + Q(inactive_individuals__family__in=remaining_families) | Q(active_individuals__family__in=remaining_families) + ).distinct() + logger.info(f'Splitting {split_dataset.count()} datasets') + for dataset in split_dataset: + individuals = dataset.inactive_individuals.filter(family__in=families) + dataset.inactive_individuals.remove(*individuals) + new_dataset = dataset + new_dataset.pk = None + new_dataset.save() + new_dataset.inactive_individuals.set(individuals) + for variant_tag_type in VariantTagType.objects.filter(project=from_project): variant_tags = VariantTag.objects.filter(saved_variants__family__in=families, variant_tag_type=variant_tag_type) if variant_tags: diff --git a/seqr/management/commands/update_individuals_sample_qc.py b/seqr/management/commands/update_individuals_sample_qc.py index fcb3e7fcf5..eb9e5d6889 100644 --- a/seqr/management/commands/update_individuals_sample_qc.py +++ b/seqr/management/commands/update_individuals_sample_qc.py @@ -4,7 +4,7 @@ from django.core.management.base import BaseCommand, CommandError from reference_data.models import GENOME_VERSION_LOOKUP, GENOME_VERSION_GRCh38 -from seqr.models import Sample +from seqr.models import Dataset from seqr.management.commands.check_for_new_samples_from_pipeline import update_individuals_sample_qc, get_pipeline_runs from seqr.utils.file_utils import file_iter @@ -13,7 +13,7 @@ class Command(BaseCommand): help = 'Ingest sample qc data for a particular pipeline run' def add_arguments(self, parser): - parser.add_argument('dataset_type', choices={Sample.DATASET_TYPE_VARIANT_CALLS}) + parser.add_argument('dataset_type', choices={Dataset.DATASET_TYPE_VARIANT_CALLS}) parser.add_argument('genome_version', choices={GENOME_VERSION_LOOKUP[GENOME_VERSION_GRCh38]}) parser.add_argument('run_version') diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 55e19a2553..7ae407db96 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -6,27 +6,20 @@ import mock import responses -from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, AuthenticationTestCase, DifferentDbTransactionSupportMixin -from seqr.models import Project, Family, Individual, Sample, SavedVariant +from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, AuthenticationTestCase +from seqr.models import Project, Family, Individual, Dataset, SavedVariant SEQR_URL = 'https://seqr.broadinstitute.org/' PROJECT_GUID = 'R0003_test' EXTERNAL_PROJECT_GUID = 'R0004_non_analyst_project' -GUID_ID = 54321 -GCNV_GUID_ID = 12345 -NEW_SAMPLE_GUID_P3 = f'S00000{GUID_ID}_na20888' -NEW_SAMPLE_GUID_P4 = f'S00000{GUID_ID}_na21234' -REPLACED_SAMPLE_GUID = f'S00000{GUID_ID}_na20885' -EXISTING_INACTIVE_SAMPLE_GUID = 'S000154_na20889' -ACTIVE_SAMPLE_GUID = f'S00000{GUID_ID}_na20889' -EXISTING_WGS_SAMPLE_GUID = 'S000144_na20888' -EXISTING_SV_SAMPLE_GUID = 'S000147_na21234' -SAMPLE_GUIDS = [ACTIVE_SAMPLE_GUID, REPLACED_SAMPLE_GUID, NEW_SAMPLE_GUID_P3, NEW_SAMPLE_GUID_P4] -GCNV_SAMPLE_GUID = f'S00000{GCNV_GUID_ID}_na20889' -EXISTING_GCNV_SAMPLE_GUIDS = ['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733'] -GCNV_SAMPLE_GUIDS = [f'S00000{GCNV_GUID_ID}_hg00731', f'S00000{GCNV_GUID_ID}_hg00732', f'S00000{GCNV_GUID_ID}_hg00733', GCNV_SAMPLE_GUID] -OLD_DATA_SAMPLE_GUID = 'S000143_na20885' +EXISTING_INACTIVE_DATASET_GUID = 'S000154_na20889' +EXISTING_WGS_DATASET_GUID = 'S000144_na20888' +EXISTING_SV_DATASET_GUID = 'S000147_na21234' +EXISTING_GCNV_DATASET_GUID = 'S000145_hg00731' +OLD_DATA_DATASET_GUID = 'S000143_na20885' +NEW_DATASET_GUIDS = ['D0000155_snv_indel_wes_2025_09', 'D0000156_snv_indel_wes_2025_09'] +NEW_GCNV_DATASET_GUIDS = ['D0000157_sv_wes_2025_09_23_000', 'D0000158_sv_wes_2025_09_23_000'] namespace_path = 'ext-data/anvil-non-analyst-project 1000 Genomes Demo' anvil_link = f'{namespace_path}' @@ -384,14 +377,14 @@ def set_up(self): self.mock_redis = patcher.start() self.mock_redis.return_value.keys.side_effect = lambda pattern: [pattern] self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.models.random.randint') - mock_rand_int = patcher.start() - mock_rand_int.side_effect = [GUID_ID, GUID_ID, GUID_ID, GUID_ID, GCNV_GUID_ID, GCNV_GUID_ID, GCNV_GUID_ID, GCNV_GUID_ID, GUID_ID, GUID_ID, GUID_ID, GUID_ID] - self.addCleanup(patcher.stop) patcher = mock.patch('seqr.management.commands.check_for_new_samples_from_pipeline.PIPELINE_DATA_DIR') mock_data_dir = patcher.start() mock_data_dir.__str__.return_value = self.MOCK_DATA_DIR self.addCleanup(patcher.stop) + patcher = mock.patch('seqr.management.commands.check_for_new_samples_from_pipeline.timezone.now') + mock_now = patcher.start() + mock_now.return_value = datetime(2025, 9, 23) + self.addCleanup(patcher.stop) patcher = mock.patch('seqr.views.utils.export_utils.TemporaryDirectory') mock_temp_dir = patcher.start() mock_temp_dir.return_value.__enter__.return_value = '/mock/tmp' @@ -405,13 +398,13 @@ def set_up(self): patcher = mock.patch('seqr.utils.communication_utils.BASE_URL', SEQR_URL) patcher.start() self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.utils.search.add_data_utils.BASE_URL', SEQR_URL) + patcher = mock.patch('seqr.utils.add_data_utils.BASE_URL', SEQR_URL) patcher.start() self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.utils.search.add_data_utils.SEQR_SLACK_ANVIL_DATA_LOADING_CHANNEL', 'anvil-data-loading') + patcher = mock.patch('seqr.utils.add_data_utils.SEQR_SLACK_ANVIL_DATA_LOADING_CHANNEL', 'anvil-data-loading') patcher.start() self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.utils.search.add_data_utils.SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL', 'seqr-data-loading') + patcher = mock.patch('seqr.utils.add_data_utils.SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL', 'seqr-data-loading') patcher.start() self.addCleanup(patcher.stop) patcher = mock.patch('seqr.views.utils.airtable_utils.BASE_URL', 'https://test-seqr.org/') @@ -421,7 +414,7 @@ def set_up(self): patcher.start() self.addCleanup(patcher.stop) - Sample.objects.filter(guid=OLD_DATA_SAMPLE_GUID).update(sample_type='WES') + Dataset.objects.filter(guid=OLD_DATA_DATASET_GUID).update(sample_type='WES') def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=6): self._set_loading_files() @@ -463,16 +456,16 @@ def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=6): num_calls = self._assert_expected_airtable_calls(bool(run_loading_logs), single_call) self.assertEqual(len(responses.calls), num_calls) - def _test_success_call(self, anvil_email_calls): + def _test_success_call(self, anvil_email_calls, next_dataset_id=155): Project.objects.filter(id__in=[1, 3]).update(genome_version=38) + self.maxDiff = None self._test_call(run_loading_logs={ 'GRCh38/SNV_INDEL': [ ('Loading 4 WES SNV_INDEL samples in 2 projects', None), ('update 2 Familys', {'dbUpdate': mock.ANY}), - ('create 4 Samples', {'dbUpdate': mock.ANY}), - ('update 4 Samples', {'dbUpdate': mock.ANY}), - ('update 1 Samples', {'dbUpdate': mock.ANY}), + (f'create Dataset D0000{next_dataset_id}_snv_indel_wes_2025_09', {'dbUpdate': mock.ANY}), + (f'create Dataset D0000{next_dataset_id+1}_snv_indel_wes_2025_09', {'dbUpdate': mock.ANY}), ] + self.AIRTABLE_LOGS + [ ('update 3 Familys', {'dbUpdate': mock.ANY}), ] + self.UPDATE_SAMPLE_LOGS, @@ -482,9 +475,8 @@ def _test_success_call(self, anvil_email_calls): 'GRCh38/SV': [ ('Loading 4 WES SV samples in 2 projects', None), ('update 1 Familys', {'dbUpdate': mock.ANY}), - ('create 4 Samples', {'dbUpdate': mock.ANY}), - ('update 4 Samples', {'dbUpdate': mock.ANY}), - ('update 3 Samples', {'dbUpdate': mock.ANY}), + (f'create Dataset D0000{next_dataset_id+2}_sv_wes_2025_09_23_000', {'dbUpdate': mock.ANY}), + (f'create Dataset D0000{next_dataset_id+3}_sv_wes_2025_09_23_000', {'dbUpdate': mock.ANY}), ('Reloading saved variants in 2 projects', None), ('Updated 0 variants in 1 families for project 1kg project nåme with uniçøde', None), ('Updated 0 variants in 1 families for project Test Reprocessed Project', None), @@ -561,11 +553,15 @@ def _test_success_call(self, anvil_email_calls): self.assertDictEqual(self.mock_email.return_value.merge_data, {}) self.assertEqual(self.manager_user.notifications.count(), 5) - self.assertEqual( - str(self.manager_user.notifications.first()), 'Test Reprocessed Project Loaded 1 new WES SV samples 0 minutes ago') + manager_notification = self.manager_user.notifications.order_by('-id').first() + self.assertEqual(manager_notification.actor.name, 'Test Reprocessed Project') + self.assertEqual(manager_notification.verb, 'Loaded 1 new WES SV samples') + self.assertEqual(manager_notification.timestamp.isoformat(), '2025-09-23T00:00:00+00:00') self.assertEqual(self.collaborator_user.notifications.count(), 2) - self.assertEqual( - str(self.collaborator_user.notifications.first()), 'Non-Analyst Project Loaded 1 new WES samples 0 minutes ago') + collaborator_notification = self.collaborator_user.notifications.first() + self.assertEqual(collaborator_notification.actor.name, 'Non-Analyst Project') + self.assertEqual(collaborator_notification.verb, 'Loaded 1 new WES samples') + self.assertEqual(collaborator_notification.timestamp.isoformat(), '2025-09-23T00:00:00+00:00') def _additional_loading_logs(self, data_type, version): return [] @@ -596,7 +592,7 @@ def test_command(self): 'auto__2024-09-14': 'Data has genome version GRCh38 but the following projects have conflicting versions: R0001_1kg (GRCh37), R0003_test (GRCh37)', } self._test_call(error_logs=error_logs) - self.assertEqual(Sample.objects.filter(guid__in=SAMPLE_GUIDS + GCNV_SAMPLE_GUIDS).count(), 0) + self.assertEqual(Dataset.objects.filter(guid__in=NEW_DATASET_GUIDS + NEW_GCNV_DATASET_GUIDS).count(), 0) # Update fixture data to allow testing edge cases svs = SavedVariant.objects.filter(guid__in=['SV0000002_1248367227_r0390_100', 'SV0000006_1248367227_r0003_tes', 'SV0000007_prefix_19107_DEL_r00']) @@ -610,55 +606,71 @@ def test_command(self): self._test_success_call(self._anvil_email_calls()) # Tests Sample models created/updated - snv_indel_samples = Sample.objects.filter(data_source='auto__2023-08-09') - gcnv_samples = Sample.objects.filter(data_source='auto__2024-09-14') - updated_sample_models = snv_indel_samples | gcnv_samples - self.assertSetEqual({'WES'}, set(updated_sample_models.values_list('sample_type', flat=True))) - self.assertSetEqual({True}, set(updated_sample_models.values_list('is_active', flat=True))) + snv_indel_datasets = Dataset.objects.filter(data_source='auto__2023-08-09') + gcnv_datasets = Dataset.objects.filter(data_source='auto__2024-09-14') + new_dataset_models = snv_indel_datasets | gcnv_datasets + self.assertSetEqual({'WES'}, set(new_dataset_models.values_list('sample_type', flat=True))) self.assertSetEqual( - {datetime.now().strftime('%Y-%m-%d')}, - {date.strftime('%Y-%m-%d') for date in updated_sample_models.values_list('loaded_date', flat=True)} + {'2025-09-23'}, + {date.strftime('%Y-%m-%d') for date in new_dataset_models.values_list('loaded_date', flat=True)} ) - self.assertSetEqual(set(snv_indel_samples.values_list('guid', flat=True)), set(SAMPLE_GUIDS)) - self.assertSetEqual({'SNV_INDEL'}, set(snv_indel_samples.values_list('dataset_type', flat=True))) - self.assertSetEqual({'1kg.vcf.gz;new_samples.vcf.gz'}, set(snv_indel_samples.values_list('elasticsearch_index', flat=True))) + self.assertSetEqual(set(snv_indel_datasets.values_list('guid', flat=True)), set(NEW_DATASET_GUIDS)) + self.assertSetEqual({'SNV_INDEL'}, set(snv_indel_datasets.values_list('dataset_type', flat=True))) - self.assertSetEqual(set(gcnv_samples.values_list('guid', flat=True)), set(GCNV_SAMPLE_GUIDS)) - self.assertSetEqual({'SV'}, set(gcnv_samples.values_list('dataset_type', flat=True))) - self.assertSetEqual({'gcnv.bed.gz'}, set(gcnv_samples.values_list('elasticsearch_index', flat=True))) + self.assertSetEqual(set(gcnv_datasets.values_list('guid', flat=True)), set(NEW_GCNV_DATASET_GUIDS)) + self.assertSetEqual({'SV'}, set(gcnv_datasets.values_list('dataset_type', flat=True))) - self.assertFalse(Sample.objects.get(guid=OLD_DATA_SAMPLE_GUID).is_active) + old_dataset = Dataset.objects.get(guid=OLD_DATA_DATASET_GUID) + self.assertListEqual(list(old_dataset.active_individuals.values_list('id', flat=True)), []) + self.assertListEqual(list(old_dataset.inactive_individuals.values_list('id', flat=True)), [15]) - previous_gcnv_samples = Sample.objects.filter(guid__in=EXISTING_GCNV_SAMPLE_GUIDS) - self.assertEqual(len(previous_gcnv_samples), len(EXISTING_GCNV_SAMPLE_GUIDS)) - self.assertFalse(any(previous_gcnv_samples.values_list('is_active', flat=True))) + old_gcnv_dataset = Dataset.objects.get(guid=EXISTING_GCNV_DATASET_GUID) + self.assertListEqual(list(old_gcnv_dataset.active_individuals.values_list('id', flat=True)), []) + self.assertListEqual(list(old_gcnv_dataset.inactive_individuals.values_list('id', flat=True)), [4, 5, 6]) # Previously loaded WGS data should be unchanged by loading WES data self.assertEqual( - Sample.objects.get(guid=EXISTING_WGS_SAMPLE_GUID).last_modified_date.strftime('%Y-%m-%d'), '2017-03-13') + Dataset.objects.get(guid=EXISTING_WGS_DATASET_GUID).last_modified_date.strftime('%Y-%m-%d'), '2017-03-13') # Previously loaded SV data should be unchanged by loading SNV_INDEL data - sv_sample = Sample.objects.get(guid=EXISTING_SV_SAMPLE_GUID) - self.assertEqual(sv_sample.last_modified_date.strftime('%Y-%m-%d'), '2018-03-13') - self.assertTrue(sv_sample.is_active) + sv_dataset= Dataset.objects.get(guid=EXISTING_SV_DATASET_GUID) + self.assertEqual(sv_dataset.last_modified_date.strftime('%Y-%m-%d'), '2018-03-13') + self.assertListEqual(list(sv_dataset.active_individuals.values_list('id', flat=True)), [18, 19, 21]) + self.assertListEqual(list(sv_dataset.inactive_individuals.values_list('id', flat=True)), []) # Test Individual models properly associated with Samples self.assertSetEqual( - set(Individual.objects.get(guid='I000015_na20885').sample_set.values_list('guid', flat=True)), - {REPLACED_SAMPLE_GUID, OLD_DATA_SAMPLE_GUID} + set(Individual.objects.get(guid='I000015_na20885').active_datasets.values_list('guid', flat=True)), + {NEW_DATASET_GUIDS[0]} + ) + self.assertSetEqual( + set(Individual.objects.get(guid='I000015_na20885').inactive_datasets.values_list('guid', flat=True)), + {OLD_DATA_DATASET_GUID} + ) + self.assertSetEqual( + set(Individual.objects.get(guid='I000016_na20888').active_datasets.values_list('guid', flat=True)), + {NEW_DATASET_GUIDS[0]} + ) + self.assertSetEqual( + set(Individual.objects.get(guid='I000016_na20888').inactive_datasets.values_list('guid', flat=True)), + {EXISTING_WGS_DATASET_GUID} + ) + self.assertSetEqual( + set(Individual.objects.get(guid='I000017_na20889').active_datasets.values_list('guid', flat=True)), + {NEW_DATASET_GUIDS[0], NEW_GCNV_DATASET_GUIDS[1]} ) self.assertSetEqual( - set(Individual.objects.get(guid='I000016_na20888').sample_set.values_list('guid', flat=True)), - {EXISTING_WGS_SAMPLE_GUID, NEW_SAMPLE_GUID_P3} + set(Individual.objects.get(guid='I000017_na20889').inactive_datasets.values_list('guid', flat=True)), + {EXISTING_INACTIVE_DATASET_GUID} ) self.assertSetEqual( - set(Individual.objects.get(guid='I000017_na20889').sample_set.values_list('guid', flat=True)), - {EXISTING_INACTIVE_SAMPLE_GUID, ACTIVE_SAMPLE_GUID, GCNV_SAMPLE_GUID} + set(Individual.objects.get(guid='I000018_na21234').active_datasets.values_list('guid', flat=True)), + {NEW_DATASET_GUIDS[1], EXISTING_SV_DATASET_GUID} ) self.assertSetEqual( - set(Individual.objects.get(guid='I000018_na21234').sample_set.values_list('guid', flat=True)), - {EXISTING_SV_SAMPLE_GUID, NEW_SAMPLE_GUID_P4} + set(Individual.objects.get(guid='I000018_na21234').inactive_datasets.values_list('guid', flat=True)), + set(), ) # Test Individual model properly updated with sample qc results @@ -707,26 +719,26 @@ def test_command(self): self.mock_email.reset_mock() self.mock_send_slack.reset_mock() self.mock_redis.reset_mock() - sample_last_modified = Sample.objects.filter( + dataset_last_modified = Dataset.objects.filter( last_modified_date__isnull=False).values_list('last_modified_date', flat=True).order_by('-last_modified_date')[0] call_command('check_for_new_samples_from_pipeline') self.assert_json_logs(user=None, expected=self.LIST_FILE_LOGS[:1] + [('Data already loaded for all 2 runs', None)]) self.mock_email.assert_not_called() self.mock_send_slack.assert_not_called() - self.assertFalse(Sample.objects.filter(last_modified_date__gt=sample_last_modified).exists()) + self.assertFalse(Dataset.objects.filter(last_modified_date__gt=dataset_last_modified).exists()) self.mock_redis.return_value.delete.assert_not_called() # Test reloading shared annotations is skipped if too many saved variants - snv_indel_samples.delete() + snv_indel_datasets.delete() airtable_logs = self.AIRTABLE_LOGS[:-1] if self.AIRTABLE_LOGS: airtable_logs.append(('Fetched 1 AnVIL Seqr Loading Requests Tracking records from airtable', None)) self._test_call(num_runs=2, run_loading_logs={ 'GRCh38/SNV_INDEL': [ ('Loading 4 WES SNV_INDEL samples in 2 projects', None), - ('create 4 Samples', {'dbUpdate': mock.ANY}), - ('update 4 Samples', {'dbUpdate': mock.ANY}), + ('create Dataset D0000159_snv_indel_wes_2025_09', {'dbUpdate': mock.ANY}), + ('create Dataset D0000160_snv_indel_wes_2025_09', {'dbUpdate': mock.ANY}), ] + airtable_logs + self.UPDATE_SAMPLE_LOGS, }) @@ -1030,4 +1042,4 @@ def test_loading_delay_command(self): self._test_success_call(self._anvil_email_calls( email_text=ANVIL_ERROR_TEXT_EMAIL_TEMPLATE.format(error='\n'+ANVIL_ERROR_DELAY), email_html=ANVIL_ERROR_HTML_EMAIL_TEMPLATE.format(error='
'+ANVIL_ERROR_DELAY), - )) \ No newline at end of file + ), next_dataset_id=161) \ No newline at end of file diff --git a/seqr/management/tests/reload_saved_variant_genotypes_tests.py b/seqr/management/tests/reload_saved_variant_genotypes_tests.py index 4f7f8cf3e9..50ec2d6003 100644 --- a/seqr/management/tests/reload_saved_variant_genotypes_tests.py +++ b/seqr/management/tests/reload_saved_variant_genotypes_tests.py @@ -1,7 +1,7 @@ from django.core.management import call_command from seqr.views.utils.test_utils import AnvilAuthenticationTestCase -from seqr.models import SavedVariant, Sample +from seqr.models import SavedVariant, Dataset class ReloadSavedVariantGenotypesTest(AnvilAuthenticationTestCase): @@ -9,7 +9,8 @@ class ReloadSavedVariantGenotypesTest(AnvilAuthenticationTestCase): def test_command(self): # Update fixture data - Sample.objects.filter(id__in=[143, 149]).update(individual_id=18, sample_id='NA21234') + for dataset in Dataset.objects.filter(id__in=[143, 149]): + dataset.active_individuals.set([18]) call_command('reload_saved_variant_genotypes', 'R0004_non_analyst_project') self.assert_json_logs(user=None, expected=[ diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index f03bd95a59..34c77cf5e1 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -3,7 +3,7 @@ import json import mock -from clickhouse_search.search_tests import ClickhouseSearchTestCase +from clickhouse_search.all_search_tests import ClickhouseSearchTestCase from clickhouse_search.test_utils import VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT3, GCNV_VARIANT4 from seqr.models import SavedVariant, VariantTag diff --git a/seqr/management/tests/transfer_families_to_different_project_tests.py b/seqr/management/tests/transfer_families_to_different_project_tests.py index b2feeecd53..d346420ad0 100644 --- a/seqr/management/tests/transfer_families_to_different_project_tests.py +++ b/seqr/management/tests/transfer_families_to_different_project_tests.py @@ -1,7 +1,7 @@ from django.core.management import call_command import responses -from seqr.models import Family, VariantTagType, VariantTag, Sample +from seqr.models import Family, VariantTagType, VariantTag, Dataset from seqr.views.utils.test_utils import AnvilAuthenticationTestCase @@ -26,6 +26,7 @@ def test_command(self): ('Found 3 out of 4 families. No match for: 12.', None), ('Skipping 1 families with analysis groups in the project: 5 (Test Group 1)', None), *self.LOGS, + ('Splitting 2 datasets', None), ('Updating "Excluded" tags', None), ('Updating families', None), ('Done.', None), @@ -45,9 +46,21 @@ def test_command(self): self.assertEqual(len(new_tags), 1) self.assertEqual(new_tags[0].saved_variants.first().family, family) - samples = Sample.objects.filter(individual__family=family) - self.assertEqual(samples.count(), 7) - self.assertEqual(samples.filter(is_active=True).count(), 0) + existing_dataset = Dataset.objects.get(guid='S000129_na19675') + self.assertListEqual(list(existing_dataset.active_individuals.order_by('id').values_list('id', flat=True)),[1, 7, 9]) + self.assertListEqual(list(existing_dataset.inactive_individuals.order_by('id').values_list('id', flat=True)), [3]) + self.assertEqual(Dataset.objects.filter(active_individuals__family=family).count(), 0) + datasets = {d.guid: d for d in Dataset.objects.filter(inactive_individuals__family=family).distinct()} + self.assertEqual(len(datasets), 3) + previous_guids = {'S000145_hg00731', 'S000149_hg00733'} + self.assertTrue(previous_guids.issubset(set(datasets.keys()))) + split_dataset = next(d for guid, d in datasets.items() if guid not in previous_guids) + self.assertListEqual(list(split_dataset.inactive_individuals.order_by('id').values_list('id', flat=True)), [4, 5, 6]) + self.assertEqual(split_dataset.active_individuals.count(), 0) + self.assertEqual(split_dataset.sample_type, 'WES') + self.assertEqual(split_dataset.dataset_type, 'SNV_INDEL') + self.assertEqual(split_dataset.data_source, 'test_index') + self.assertEqual(split_dataset.loaded_date.isoformat(), '2017-02-05T06:12:55.397000+00:00') family = Family.objects.get(family_id='4') self.assertEqual(family.project.guid, 'R0003_test') diff --git a/seqr/migrations/0087_dataset.py b/seqr/migrations/0087_dataset.py new file mode 100644 index 0000000000..1ae541adf6 --- /dev/null +++ b/seqr/migrations/0087_dataset.py @@ -0,0 +1,117 @@ +# Generated by Django 4.2.24 on 2026-04-06 16:39 + +from django.conf import settings +from django.contrib.postgres.aggregates import ArrayAgg +from django.db import migrations, models +from django.db.models.functions import Trim, TruncDate +import django.db.models.deletion +import django.utils.timezone + + +def set_individual_id(apps, schema_editor): + Sample = apps.get_model('seqr', 'Sample') + Individual = apps.get_model('seqr', 'Individual') + + db_alias = schema_editor.connection.alias + + mismatch_samples = Sample.objects.using(db_alias).filter(is_active=True).exclude(sample_id=models.F('individual__individual_id')) + individual_ids = [] + for sample in mismatch_samples: + individual = sample.individual + individual.display_name = individual.individual_id + # Since some of these are sample swamps, set a padded id to avoid conflicts and then trim whitespace for clean up + individual.individual_id = f' {sample.sample_id} ' + individual.save() + individual_ids.append(individual.id) + + updated = Individual.objects.using(db_alias).filter(id__in=individual_ids).update(individual_id=Trim('individual_id')) + if updated: + print(f'Updated individual_id for {updated} individuals to match their active sample_id') + + +def populate_datasets(apps, schema_editor): + Dataset = apps.get_model('seqr', 'Dataset') + Sample = apps.get_model('seqr', 'Sample') + db_alias = schema_editor.connection.alias + + dataset_aggs = Sample.objects.using(db_alias).values( + 'sample_type', 'dataset_type', project=models.F('individual__family__project'), group_loaded_date=TruncDate('loaded_date'), + ).annotate( + guid=models.Max('guid'), + loaded_date=models.Max('loaded_date'), + created_date=models.Max('created_date'), + last_modified_date=models.Max('last_modified_date'), + created_by_id=models.Min('created_by_id'), + data_sources=ArrayAgg('data_source', distinct=True, filter=models.Q(data_source__isnull=False)), + es_indices=ArrayAgg('elasticsearch_index', distinct=True, filter=models.Q(elasticsearch_index__isnull=False)), + active_individuals=ArrayAgg('individual__id', distinct=True, filter=models.Q(is_active=True)), + inactive_individuals=ArrayAgg('individual__id', distinct=True, filter=models.Q(is_active=False)), + ) + + datasets = [] + active_individuals = [] + inactive_individuals = [] + for dataset_agg in dataset_aggs: + del dataset_agg['project'] + del dataset_agg['group_loaded_date'] + active_individuals.append(dataset_agg.pop('active_individuals')) + inactive_individuals.append(dataset_agg.pop('inactive_individuals')) + data_sources = dataset_agg.pop('data_sources') + es_indices = dataset_agg.pop('es_indices') + datasets.append(Dataset(**dataset_agg, data_source=','.join(data_sources or es_indices))) + + created = Dataset.objects.using(db_alias).bulk_create(datasets) + if created: + print(f'Created {len(created)} datasets.') + + ActiveRelation = Dataset.active_individuals.through + InactiveRelation = Dataset.inactive_individuals.through + + active_relations = [] + for dataset, individual_ids in zip(created, active_individuals): + active_relations.extend(ActiveRelation(dataset_id=dataset.id, individual_id=ind_id) for ind_id in individual_ids) + ActiveRelation.objects.using(db_alias).bulk_create(active_relations) + if active_relations: + print(f'Created {len(active_relations)} active samples relations') + + inactive_relations = [] + for dataset, individual_ids in zip(created, inactive_individuals): + inactive_relations.extend(InactiveRelation(dataset_id=dataset.id, individual_id=ind_id) for ind_id in individual_ids) + InactiveRelation.objects.using(db_alias).bulk_create(inactive_relations) + if inactive_relations: + print(f'Created {len(inactive_relations)} inactive samples relations') + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('seqr', '0086_alter_rnasample_tissue_type'), + ] + + operations = [ + migrations.CreateModel( + name='Dataset', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('guid', models.CharField(db_index=True, max_length=30, unique=True)), + ('created_date', models.DateTimeField(db_index=True, default=django.utils.timezone.now)), + ('last_modified_date', models.DateTimeField(blank=True, db_index=True, null=True)), + ('sample_type', models.CharField(choices=[('WES', 'Exome'), ('WGS', 'Whole Genome')], max_length=10)), + ('dataset_type', models.CharField(choices=[('SNV_INDEL', 'Variant Calls'), ('SV', 'SV Calls'), ('MITO', 'Mitochondria calls')], max_length=13)), + ('data_source', models.TextField()), + ('loaded_date', models.DateTimeField()), + ('active_individuals', models.ManyToManyField(related_name='active_datasets', to='seqr.individual')), + ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)), + ('inactive_individuals', models.ManyToManyField(related_name='inactive_datasets', to='seqr.individual')), + ], + options={ + 'json_fields': ['guid', 'sample_type', 'dataset_type', 'loaded_date'], + }, + ), + migrations.RunPython(set_individual_id), + migrations.RunPython(populate_datasets), + migrations.DeleteModel( + name='Sample', + ), + ] diff --git a/seqr/migrations/0088_rnasample_sequencing_type.py b/seqr/migrations/0088_rnasample_sequencing_type.py new file mode 100644 index 0000000000..dddada36e7 --- /dev/null +++ b/seqr/migrations/0088_rnasample_sequencing_type.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.24 on 2026-04-30 16:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('seqr', '0087_dataset'), + ] + + operations = [ + migrations.AddField( + model_name='rnasample', + name='sequencing_type', + field=models.CharField(choices=[('K', 'Kinnex'), ('T', 'Tru-Seq'), ('W', 'Watchmaker')], default='T', max_length=1), + preserve_default=False, + ), + ] diff --git a/seqr/models.py b/seqr/models.py index 4b625edc7a..149dd30655 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -85,16 +85,12 @@ def __unicode__(self): return self.guid def __str__(self): - """Magix function for str() and %s.""" return self.__unicode__() def json(self): - """Utility method that returns a json {field-name: value-as-string} mapping for all fields.""" return {k: v for k, v in self.__dict__.items() if not k.startswith('_')} def save(self, *args, **kwargs): - """Create a GUID at object creation time.""" - being_created = not self.pk current_time = timezone.now() @@ -113,11 +109,11 @@ def save(self, *args, **kwargs): self.created_date = kwargs.pop('created_date', current_time) super(ModelWithGUID, self).save(*args, **kwargs) + # Create a GUID at object creation time self.guid = self._compute_guid() super(ModelWithGUID, self).save() def delete_model(self, user, user_can_delete=False): - """Helper delete method that logs the deletion""" if not (user_can_delete or self.created_by == user): raise PermissionDenied('User does not have permission to delete this {}'.format(type(self).__name__)) self.delete() @@ -125,7 +121,6 @@ def delete_model(self, user, user_can_delete=False): @classmethod def bulk_create(cls, user, new_models, **kwargs): - """Helper bulk create method that logs the creation""" for model in new_models: model.created_by = user model.created_date = timezone.now() @@ -136,7 +131,6 @@ def bulk_create(cls, user, new_models, **kwargs): @classmethod def bulk_update(cls, user, update_json, queryset=None, **filter_kwargs): - """Helper bulk update method that logs the update""" if queryset is None: queryset = cls.objects.filter(**filter_kwargs).exclude(**update_json) @@ -149,13 +143,11 @@ def bulk_update(cls, user, update_json, queryset=None, **filter_kwargs): @classmethod def bulk_update_models(cls, user, models, fields): - """Helper bulk update method that logs the update and allows different update data for each model""" log_model_bulk_update(logger, models, user, 'update', update_fields=fields) cls.objects.bulk_update(models, fields) @classmethod def bulk_delete(cls, user, queryset=None, **filter_kwargs): - """Helper bulk delete method that logs the deletion""" if queryset is None: queryset = cls.objects.filter(**filter_kwargs) log_model_bulk_update(logger, queryset, user, 'delete') @@ -219,10 +211,6 @@ def __unicode__(self): GUID_PRECISION = 4 def save(self, *args, **kwargs): - """Override the save method and create user permissions groups + add the created_by user. - - This could be done with signals, but seems cleaner to do it this way. - """ being_created = not self.pk anvil_disabled = not anvil_enabled() @@ -249,8 +237,6 @@ def save(self, *args, **kwargs): user.groups.add(*[getattr(self, group) for group in groups]) def delete(self, *args, **kwargs): - """Override the delete method to also delete the project-specific user groups""" - super(Project, self).delete(*args, **kwargs) if self.can_edit_group: @@ -616,7 +602,6 @@ class Individual(ModelWithGUID): sex = models.CharField(max_length=3, choices=SEX_CHOICES, default='U') affected = models.CharField(max_length=1, choices=AFFECTED_STATUS_CHOICES, default=AFFECTED_STATUS_UNKNOWN) - # TODO once sample and individual ids are fully decoupled no reason to maintain this field display_name = models.TextField(default="", blank=True) notes = models.TextField(blank=True, null=True) @@ -695,14 +680,7 @@ class Meta: audit_fields = {'case_review_status'} -class Sample(ModelWithGUID): - """This model represents a single data type (eg. Variant Calls, or SV Calls) that's generated from a single - biological sample (eg. WES, WGS). - - It stores metadata on both the dataset (fields: dataset_type, loaded_date, etc.) and the underlying sample - (fields: sample_type, sample_id etc.) - """ - +class Dataset(ModelWithGUID): SAMPLE_TYPE_WES = 'WES' SAMPLE_TYPE_WGS = 'WGS' SAMPLE_TYPE_CHOICES = ( @@ -721,31 +699,22 @@ class Sample(ModelWithGUID): ) DATASET_TYPE_LOOKUP = dict(DATASET_TYPE_CHOICES) - individual = models.ForeignKey('Individual', on_delete=models.PROTECT) + active_individuals = models.ManyToManyField('Individual', related_name='active_datasets') + inactive_individuals = models.ManyToManyField('Individual', related_name='inactive_datasets') sample_type = models.CharField(max_length=10, choices=SAMPLE_TYPE_CHOICES) dataset_type = models.CharField(max_length=13, choices=DATASET_TYPE_CHOICES) - # The sample's id in the underlying dataset (eg. the VCF Id for variant callsets). - sample_id = models.TextField(db_index=True) - - elasticsearch_index = models.TextField(db_index=True, null=True) - data_source = models.TextField(null=True) - - # sample status - is_active = models.BooleanField(default=False) + data_source = models.TextField() loaded_date = models.DateTimeField() def __unicode__(self): - return self.sample_id.strip() + return f'{self.dataset_type}_{self.sample_type}_{self.loaded_date}' - GUID_PREFIX = 'S' - GUID_PRECISION = 10 + GUID_PREFIX = 'D' class Meta: - json_fields = [ - 'guid', 'created_date', 'sample_type', 'dataset_type', 'sample_id', 'is_active', 'loaded_date', - ] + json_fields = ['guid', 'sample_type', 'dataset_type', 'loaded_date'] class RnaSample(ModelWithGUID): @@ -771,10 +740,17 @@ class RnaSample(ModelWithGUID): ('S', 'skin'), ) + SEQUENCING_CHOICES = ( + ('K', 'Kinnex'), + ('T', 'Tru-Seq'), + ('W', 'Watchmaker'), + ) + individual = models.ForeignKey('Individual', on_delete=models.PROTECT) data_type = models.CharField(max_length=1, choices=DATA_TYPE_CHOICES) tissue_type = models.CharField(max_length=2, choices=TISSUE_TYPE_CHOICES) + sequencing_type = models.CharField(max_length=1, choices=SEQUENCING_CHOICES) data_source = models.TextField() is_active = models.BooleanField(default=False) @@ -788,9 +764,6 @@ class Meta: class IgvSample(ModelWithGUID): - """This model represents a single data type that can be displayed in IGV (eg. Read Alignments) that's generated from - a single biological sample (eg. WES, WGS, RNA, Array). - """ SAMPLE_TYPE_ALIGNMENT = 'alignment' SAMPLE_TYPE_COVERAGE = 'wig' SAMPLE_TYPE_JUNCTION = 'spliceJunctions' @@ -828,10 +801,10 @@ class Meta: class SavedVariant(ModelWithGUID): DATASET_TYPE_CHOICES = ( - (Sample.DATASET_TYPE_VARIANT_CALLS, 'Variant Calls'), - (Sample.DATASET_TYPE_MITO_CALLS, 'Mitochondria calls'), - (f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WGS}', 'SV WGS Calls'), - (f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}', 'gCNV Calls'), + (Dataset.DATASET_TYPE_VARIANT_CALLS, 'Variant Calls'), + (Dataset.DATASET_TYPE_MITO_CALLS, 'Mitochondria calls'), + (f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WGS}', 'SV WGS Calls'), + (f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WES}', 'gCNV Calls'), ) family = models.ForeignKey('Family', on_delete=models.CASCADE) @@ -867,20 +840,6 @@ class Meta: class VariantTagType(ModelWithGUID): - """ - Previous color choices: - '#1f78b4', - '#a6cee3', - '#b2df8a', - '#33a02c', - '#fdbf6f', - '#ff7f00', - '#ff0000', - '#cab2d6', - '#6a3d9a', - '#8F754F', - '#383838', - """ project = models.ForeignKey('Project', null=True, blank=True, on_delete=models.CASCADE) name = models.TextField() @@ -1055,8 +1014,6 @@ class Meta: class LocusList(ModelWithGUID): - """List of gene ids or regions""" - name = models.TextField(db_index=True) description = models.TextField(null=True, blank=True) @@ -1186,7 +1143,6 @@ def log_model_no_guid_bulk_update(cls, models, user, update_type): @classmethod def bulk_create(cls, user, new_models, **kwargs): - """Helper bulk create method that logs the creation""" for model in new_models: model.created_by = user models = cls.objects.bulk_create(new_models, **kwargs) @@ -1195,7 +1151,6 @@ def bulk_create(cls, user, new_models, **kwargs): @classmethod def bulk_delete(cls, user, queryset=None, **filter_kwargs): - """Helper bulk delete method that logs the deletion""" if queryset is None: queryset = cls.objects.filter(**filter_kwargs) cls.log_model_no_guid_bulk_update(queryset, user, 'delete') diff --git a/seqr/urls.py b/seqr/urls.py index ab0b4b461f..5c6bf5cef7 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -1,8 +1,3 @@ -"""seqr URL Configuration - -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/1.9/topics/http/urls/ -""" from seqr.views.react_app import main_app, no_login_main_app from seqr.views.status import status_view from settings import ENABLE_DJANGO_DEBUG_TOOLBAR, MEDIA_ROOT, API_LOGIN_REQUIRED_URL, LOGIN_URL, DEBUG, \ @@ -144,7 +139,7 @@ from seqr.views.apis.project_api import create_project_handler, update_project_handler, delete_project_handler, \ project_page_data, project_families, project_overview, project_mme_submisssions, project_individuals, \ project_analysis_groups, update_project_workspace, project_family_notes, project_collaborators, project_locus_lists, \ - project_samples, project_notifications, mark_read_project_notifications, subscribe_project_notifications, \ + project_notifications, mark_read_project_notifications, subscribe_project_notifications, \ update_project_rna_seq, load_rna_seq_sample_data from seqr.views.apis.project_categories_api import update_project_categories_handler from seqr.views.apis.anvil_workspace_api import anvil_workspace_page, create_project_from_workspace, \ @@ -212,7 +207,6 @@ 'project/(?P[^/]+)/details': project_page_data, 'project/(?P[^/]+)/get_families': project_families, 'project/(?P[^/]+)/get_individuals': project_individuals, - 'project/(?P[^/]+)/get_samples': project_samples, 'project/(?P[^/]+)/get_family_notes': project_family_notes, 'project/(?P[^/]+)/get_mme_submissions': project_mme_submisssions, 'project/(?P[^/]+)/get_analysis_groups': project_analysis_groups, diff --git a/seqr/utils/search/add_data_utils.py b/seqr/utils/add_data_utils.py similarity index 92% rename from seqr/utils/search/add_data_utils.py rename to seqr/utils/add_data_utils.py index a7a5c1a2c9..2feefa04fa 100644 --- a/seqr/utils/search/add_data_utils.py +++ b/seqr/utils/add_data_utils.py @@ -5,7 +5,7 @@ import requests from reference_data.models import GeneInfo, GENOME_VERSION_LOOKUP -from seqr.models import Sample, Individual, Project +from seqr.models import Dataset, Individual, Project from seqr.utils.communication_utils import send_project_notification, safe_post_to_slack from seqr.utils.file_utils import does_file_exist from seqr.utils.logging_utils import SeqrLogger @@ -21,7 +21,7 @@ def basic_notify_search_data_loaded(project, dataset_type, sample_type, new_samples, email_template=None, is_internal=True): - msg_dataset_type = '' if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS else f' {dataset_type}' + msg_dataset_type = '' if dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS else f' {dataset_type}' num_new_samples = len(new_samples) sample_summary = f'{num_new_samples} new {sample_type}{msg_dataset_type} samples' @@ -64,12 +64,17 @@ def update_airtable_loading_tracking_status(project, status, additional_update=N ) def trigger_delete_families_search(project, family_guids, user=None): - search_samples = Sample.objects.filter(is_active=True, individual__family__guid__in=family_guids) + num_updated = 0 + updated_families = set() + for dataset in Dataset.objects.filter(active_individuals__family__guid__in=family_guids).distinct(): + active_individuals = dataset.active_individuals.filter(family__guid__in=family_guids) + num_updated += len(active_individuals) + updated_families.update(active_individuals.values_list('family__family_id', flat=True).distinct()) + dataset.inactive_individuals.add(*active_individuals) + dataset.active_individuals.remove(*active_individuals) info = [] - if search_samples: - updated_families = search_samples.values_list("individual__family__family_id", flat=True).distinct() + if num_updated: family_summary = ", ".join(sorted(updated_families)) - num_updated = search_samples.update(is_active=False) message = f'Disabled search for {num_updated} samples in the following {len(updated_families)} families: {family_summary}' info.append(message) logger.info(message, user) @@ -142,7 +147,7 @@ def _enqueue_pipeline_request(name: str, variables: dict, user: User, raise_erro def _loading_dataset_type(sample_type: str, dataset_type: str): - return 'GCNV' if dataset_type == Sample.DATASET_TYPE_SV_CALLS and sample_type == Sample.SAMPLE_TYPE_WES \ + return 'GCNV' if dataset_type == Dataset.DATASET_TYPE_SV_CALLS and sample_type == Dataset.SAMPLE_TYPE_WES \ else dataset_type diff --git a/seqr/utils/file_utils.py b/seqr/utils/file_utils.py index ce342250a2..cab280ed6f 100644 --- a/seqr/utils/file_utils.py +++ b/seqr/utils/file_utils.py @@ -83,7 +83,6 @@ def file_iter(file_path, byte_range=None, raw_content=False, user=None, **kwargs def _google_bucket_file_iter(gs_path, byte_range=None, raw_content=False, user=None, **kwargs): - """Iterate over lines in the given file""" range_arg = ' -r {}-{}'.format(byte_range[0], byte_range[1]) if byte_range else '' process = _run_gsutil_command( 'cat{}'.format(range_arg), gs_path, gunzip=gs_path.endswith("gz") and not raw_content, user=user, **kwargs) diff --git a/seqr/utils/middleware.py b/seqr/utils/middleware.py index 4f694435c9..c0dfdc5fcd 100644 --- a/seqr/utils/middleware.py +++ b/seqr/utils/middleware.py @@ -11,7 +11,7 @@ import json import traceback -from seqr.utils.search.utils import InvalidSearchException +from clickhouse_search.search import InvalidSearchException from seqr.utils.logging_utils import SeqrLogger from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.terra_api_utils import TerraAPIException diff --git a/seqr/utils/search/__init__.py b/seqr/utils/search/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py deleted file mode 100644 index 8df369fd04..0000000000 --- a/seqr/utils/search/utils.py +++ /dev/null @@ -1,172 +0,0 @@ -from collections import defaultdict -from copy import deepcopy -from datetime import timedelta - -from pyliftover.liftover import LiftOver - -from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ - get_sorted_search_results, clickhouse_variant_lookup, InvalidSearchException -from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 -from seqr.models import Sample, VariantSearchResults -from seqr.utils.logging_utils import SeqrLogger -from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json -from clickhouse_search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY -from seqr.utils.xpos_utils import parse_variant_id -from seqr.views.utils.permissions_utils import user_is_analyst - -logger = SeqrLogger(__name__) - - -MAX_EXPORT_VARIANTS = 1000 - - -def get_single_variant(families, variant_id, user): - variants = get_clickhouse_variants(families, user, raw_variant_items=variant_id, variant_ids=[variant_id]) - if not variants: - raise InvalidSearchException('Variant {} not found'.format(variant_id)) - return format_clickhouse_results(variants)[0] - - -def variant_lookup(user, variant_id, genome_version, sample_type=None, affected_only=False, hom_only=False): - cache_fields = ['variant_lookup_results', variant_id, genome_version] - if affected_only: - cache_fields.append('affected') - if hom_only: - cache_fields.append('hom') - cache_key = '__'.join(cache_fields) - variants = safe_redis_get_json(cache_key) - if variants: - return variants - - variants = clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, affected_only, hom_only) - - safe_redis_set_json(cache_key, variants, expire=timedelta(weeks=2)) - return variants - - -def _get_search_cache_key(search_model, sort=None): - return 'search_results__{}__{}'.format(search_model.guid, sort or XPOS_SORT_KEY) - - -def export_variants(search_model, user): - search_results = _query_variants(search_model, user) - total_variants = len(search_results) - if total_variants > MAX_EXPORT_VARIANTS: - raise InvalidSearchException(f'Unable to export more than {MAX_EXPORT_VARIANTS} variants ({total_variants} requested)') - return format_clickhouse_export_results(search_results) - - -def _get_previous_search_results(search_model, sort): - previous_search_results = None - if sort: - cache_key = _get_search_cache_key(search_model, sort=sort) - previous_search_results = safe_redis_get_json(cache_key) or [] - if not previous_search_results: - wildcard_cache_key = _get_search_cache_key(search_model, sort='*') - previous_search_results = safe_redis_get_wildcard_json(wildcard_cache_key) - if previous_search_results and sort: - previous_search_results = get_sorted_search_results( - previous_search_results, sort, families=search_model.families.all(), - ) - cache_key = _get_search_cache_key(search_model, sort=sort) - safe_redis_set_json(cache_key, previous_search_results, expire=timedelta(weeks=2)) - return previous_search_results - - -def query_variants(search_model, sort, page, num_results, user): - if sort == PATHOGENICTY_SORT_KEY and user_is_analyst(user): - sort = PATHOGENICTY_HGMD_SORT_KEY - all_results = _query_variants(search_model, user, sort=sort or XPOS_SORT_KEY) - - results_page = format_clickhouse_results(all_results[(page-1)*num_results:page*num_results]) - - return results_page, len(all_results) - - -def _query_variants(search_model, user, sort=None): - previous_search_results = _get_previous_search_results(search_model, sort) or {} - if previous_search_results: - return previous_search_results - - search = deepcopy(search_model.variant_search.search) - families = search_model.families.all() - - exclude_search = {} - exclude_previous_hash = (search.get('exclude') or {}).pop('previousSearchHash', None) - if exclude_previous_hash: - exclude_search = _get_clickhouse_exclude_keys(exclude_previous_hash, user) - - results = get_clickhouse_variants(families, user, sort=sort, **search, **exclude_search) - - cache_key = _get_search_cache_key(search_model, sort=sort) - safe_redis_set_json(cache_key, results, expire=timedelta(weeks=2)) - - return results - - -def _get_clickhouse_exclude_keys(search_hash, user): - previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash) - results = _query_variants(previous_search_model, user) - exclude_keys = defaultdict(list) - exclude_key_pairs = defaultdict(list) - for variant in results: - if isinstance(variant, list): - dt1= variant_dataset_type(variant[0]) - dt2 = variant_dataset_type(variant[1]) - dataset_type = dt1 if dt1 == dt2 else ','.join(sorted([dt1, dt2])) - exclude_key_pairs[dataset_type].append(sorted([variant[0]['key'], variant[1]['key']])) - else: - dataset_type = variant_dataset_type(variant) - exclude_keys[dataset_type].append(variant['key']) - return {'exclude_keys': dict(exclude_keys), 'exclude_key_pairs': dict(exclude_key_pairs)} - - -def variant_dataset_type(variant): - if not parse_variant_id(variant['variantId']): - sample_type = Sample.SAMPLE_TYPE_WGS if 'endChrom' in variant else Sample.SAMPLE_TYPE_WES - return f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' - return Sample.DATASET_TYPE_MITO_CALLS if 'mitomapPathogenic' in variant else Sample.DATASET_TYPE_VARIANT_CALLS - - -def get_variant_query_gene_counts(search_model, user): - results = _query_variants(search_model, user) - flat_variants = [ - v for variants in results for v in (variants if isinstance(variants, list) else [variants]) - ] - gene_aggs = defaultdict(lambda: {'total': 0, 'families': defaultdict(int)}) - for var in flat_variants: - gene_ids = var['transcripts'].keys() if 'transcripts' in var else {t['geneId'] for t in var['sortedTranscriptConsequences']} - for gene_id in gene_ids: - gene_aggs[gene_id]['total'] += 1 - for family_guid in var['familyGuids']: - gene_aggs[gene_id]['families'][family_guid] += 1 - return gene_aggs - - -LIFTOVERS = { - GENOME_VERSION_GRCh38: None, - GENOME_VERSION_GRCh37: None, -} -PYLIFTOVER_BUILD_LOOKUP = { - GENOME_VERSION_GRCh38: ('hg19', 'hg38'), - GENOME_VERSION_GRCh37: ('hg38', 'hg19'), -} -def _get_liftover(genome_version): - if not LIFTOVERS[genome_version]: - try: - LIFTOVERS[genome_version] = LiftOver(*PYLIFTOVER_BUILD_LOOKUP[genome_version]) - except Exception as e: - logger.error('ERROR: Unable to set up liftover. {}'.format(e), user=None) - return LIFTOVERS[genome_version] - - -def run_liftover(genome_version, chrom, pos): - liftover = _get_liftover(genome_version) - if not liftover: - return None - lifted_coord = liftover.convert_coordinate( - 'chr{}'.format(chrom.lstrip('chr')), int(pos) - ) - if lifted_coord and lifted_coord[0]: - return (lifted_coord[0][0].lstrip('chr'), lifted_coord[0][1]) - return None diff --git a/seqr/utils/social_auth_pipeline_tests.py b/seqr/utils/social_auth_pipeline_tests.py index bb08aad28f..9208004d85 100644 --- a/seqr/utils/social_auth_pipeline_tests.py +++ b/seqr/utils/social_auth_pipeline_tests.py @@ -15,7 +15,7 @@ class SocialAuthPipelineTest(TestCase): def test_validate_anvil_registration(self, mock_logger): url = TEST_TERRA_API_ROOT_URL + 'register' responses.add(responses.GET, url, status=404) - r = validate_anvil_registration(GoogleOAuth2(), {'access_token': '', 'email': 'test@seqr.org'}) + r = validate_anvil_registration(GoogleOAuth2(), {'access_token': '', 'email': 'test@seqr.org'}) # nosec mock_logger.warning.assert_called_with( 'User test@seqr.org is trying to login without registration on AnVIL. None called Terra API: GET /register got status 404 with reason: Not Found', extra={'user_email': 'test@seqr.org'} @@ -25,7 +25,7 @@ def test_validate_anvil_registration(self, mock_logger): backend = GoogleOAuth2() backend.strategy.session_set('next', '/foo/bar') - r = validate_anvil_registration(backend, {'access_token': '', 'email': 'test@seqr.org'}) + r = validate_anvil_registration(backend, {'access_token': '', 'email': 'test@seqr.org'}) # nosec self.assertEqual(r.url, '/login/error/anvil_registration?next=%2Ffoo%2Fbar') mock_logger.reset_mock() diff --git a/seqr/utils/vcf_utils.py b/seqr/utils/vcf_utils.py index 892985a490..312df0064f 100644 --- a/seqr/utils/vcf_utils.py +++ b/seqr/utils/vcf_utils.py @@ -5,7 +5,7 @@ from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.file_utils import file_iter, does_file_exist, list_files -from seqr.models import Sample +from seqr.models import Dataset BLOCK_SIZE = 65536 @@ -20,12 +20,12 @@ } DATA_TYPE_FORMAT_FIELDS = { - Sample.DATASET_TYPE_SV_CALLS: BASE_EXPECTED_FORMAT_FIELDS, + Dataset.DATASET_TYPE_SV_CALLS: BASE_EXPECTED_FORMAT_FIELDS, } DATA_TYPE_FILE_EXTS = { - Sample.DATASET_TYPE_MITO_CALLS: ('.mt',), - Sample.DATASET_TYPE_SV_CALLS: ('.bed', '.bed.gz'), + Dataset.DATASET_TYPE_MITO_CALLS: ('.mt',), + Dataset.DATASET_TYPE_SV_CALLS: ('.bed', '.bed.gz'), } REQUIRED_HEADERS = ['#CHROM', 'POS', 'ID', 'REF', 'ALT', 'QUAL', 'FILTER', 'INFO', 'FORMAT'] diff --git a/seqr/utils/xpos_utils.py b/seqr/utils/xpos_utils.py index 6d06eac680..d0619b3ccd 100644 --- a/seqr/utils/xpos_utils.py +++ b/seqr/utils/xpos_utils.py @@ -56,12 +56,6 @@ def format_chrom(chrom): def get_xpos(chrom, pos): - """Compute single number representing this chromosome and position. - - Args: - chrom (string): examples '1', 'Y', 'M' - pos (integer): genomic position on chromosome - """ chrom = format_chrom(chrom) if pos < MIN_POS or pos > MAX_POS: @@ -71,7 +65,6 @@ def get_xpos(chrom, pos): def get_chrom_pos(xpos): - """Converts xpos position to a (chr, pos) tuple""" chrom_idx = int(xpos/1e9) - 1 if chrom_idx < 0 or chrom_idx >= len(CHROMOSOMES): raise ValueError("Invalid xpos: %s" % (xpos,)) diff --git a/seqr/views/apis/anvil_workspace_api.py b/seqr/views/apis/anvil_workspace_api.py index 5106343a75..f673dc0431 100644 --- a/seqr/views/apis/anvil_workspace_api.py +++ b/seqr/views/apis/anvil_workspace_api.py @@ -1,4 +1,3 @@ -"""APIs for management of projects related to AnVIL workspaces.""" import json import time from datetime import datetime @@ -10,7 +9,7 @@ from django.shortcuts import redirect from reference_data.models import GENOME_VERSION_LOOKUP -from seqr.models import Project, Family, CAN_EDIT, Sample, IgvSample +from seqr.models import Project, Family, CAN_EDIT, Dataset, IgvSample from seqr.views.react_app import render_app_html from seqr.views.utils.airtable_utils import AirtableSession, ANVIL_REQUEST_TRACKING_TABLE from seqr.views.utils.json_to_orm_utils import create_model_from_json @@ -21,7 +20,7 @@ from seqr.views.utils.pedigree_info_utils import parse_basic_pedigree_table, JsonConstants from seqr.views.utils.individual_utils import add_or_update_individuals_and_families from seqr.utils.file_utils import list_files -from seqr.utils.search.add_data_utils import get_missing_family_samples, get_loaded_individual_ids, trigger_data_loading +from seqr.utils.add_data_utils import get_missing_family_samples, get_loaded_individual_ids, trigger_data_loading from seqr.utils.vcf_utils import validate_vcf_and_get_samples, get_vcf_list from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException @@ -58,15 +57,6 @@ def _wrapped_view(request, namespace, name, *args, **kwargs): @anvil_auth_and_policies_required(policy_url=POLICY_REQUIRED_URL) def anvil_workspace_page(request, namespace, name): - """ - This view will be requested from AnVIL, it validates the workspace and project before loading data. - - :param request: Django request object. - :param namespace: The namespace (or the billing account) of the workspace. - :param name: The name of the workspace. It also be used as the project name. - :return Redirect to a page depending on if the workspace permissions or project exists. - - """ project = Project.objects.filter(workspace_namespace=namespace, workspace_name=name) if project: return redirect('/project/{}/project_page'.format(project.first().guid)) @@ -159,15 +149,6 @@ def validate_anvil_vcf(request, namespace, name, workspace_meta): @anvil_workspace_access_required def create_project_from_workspace(request, namespace, name): - """ - Create a project when a cooperator requests to load data from an AnVIL workspace. - - :param request: Django request object - :param namespace: The namespace (or the billing account) of the workspace - :param name: The name of the workspace. It also be used as the project name - :return the projectsByGuid with the new project json - - """ projects = Project.objects.filter(workspace_namespace=namespace, workspace_name=name) if projects: error = 'Project "{}" for workspace "{}/{}" exists.'.format(projects.first().name, namespace, name) @@ -204,14 +185,6 @@ def create_project_from_workspace(request, namespace, name): @anvil_auth_and_policies_required def add_workspace_data(request, project_guid): - """ - Add data from an AnVIL workspace. - - :param request: Django request object - :param project_guid: Django request object - :return a data json with fields of individualGuid, familyGuid and optional familyNotesByGuid if no exceptions - - """ project = Project.objects.get(guid=project_guid) check_workspace_perm(request.user, CAN_EDIT, project.workspace_namespace, project.workspace_name, can_share=True) @@ -222,7 +195,7 @@ def add_workspace_data(request, project_guid): error = 'Field(s) "{}" are required'.format(', '.join(missing_fields)) return create_json_response({'error': error}, status=400, reason=error) - pedigree_records, loaded_individual_ids, sample_type = _parse_uploaded_pedigree(request_json, project=project, search_dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) + pedigree_records, loaded_individual_ids, sample_type = _parse_uploaded_pedigree(request_json, project=project, search_dataset_type=Dataset.DATASET_TYPE_VARIANT_CALLS) loading_families = {record[JsonConstants.FAMILY_ID_COLUMN] for record in pedigree_records} pending_families = Family.objects.filter( @@ -302,7 +275,7 @@ def _trigger_add_workspace_data(project, pedigree_records, user, data_path, samp f"{data_path} to seqr project <{_get_seqr_project_url(project)}|*{project.name}*> (guid: {project.guid})" ) trigger_success = trigger_data_loading( - [project], individual_ids, sample_type, Sample.DATASET_TYPE_VARIANT_CALLS, project.genome_version, data_path, user=user, success_message=success_message, + [project], individual_ids, sample_type, Dataset.DATASET_TYPE_VARIANT_CALLS, project.genome_version, data_path, user=user, success_message=success_message, success_slack_channel=SEQR_SLACK_ANVIL_DATA_LOADING_CHANNEL, error_message=f'ERROR triggering AnVIL loading for project {project.guid}', ) AirtableSession(user, base=AirtableSession.ANVIL_BASE).safe_create_records( diff --git a/seqr/views/apis/anvil_workspace_api_tests.py b/seqr/views/apis/anvil_workspace_api_tests.py index 00fd6639a4..70266db8a4 100644 --- a/seqr/views/apis/anvil_workspace_api_tests.py +++ b/seqr/views/apis/anvil_workspace_api_tests.py @@ -521,7 +521,7 @@ def _test_get_workspace_files(self, url, response_key, expected_files, mock_subp ]) -@mock.patch('seqr.utils.search.add_data_utils.LOADING_DATASETS_DIR', 'gs://seqr-loading-temp/v3.1') +@mock.patch('seqr.utils.add_data_utils.LOADING_DATASETS_DIR', 'gs://seqr-loading-temp/v3.1') @mock.patch('reference_data.models.GeneInfo.CURRENT_VERSION', '27') class LoadAnvilDataAPITest(AnvilAuthenticationTestCase, AirtableTest): fixtures = ['users', 'social_auth', 'reference_data', '1kg_project'] @@ -546,7 +546,7 @@ def setUp(self): patcher = mock.patch('seqr.views.utils.airtable_utils.logger') self.mock_airtable_logger = patcher.start() self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.utils.search.add_data_utils.logger') + patcher = mock.patch('seqr.utils.add_data_utils.logger') self.mock_add_data_utils_logger = patcher.start() self.addCleanup(patcher.stop) patcher = mock.patch('seqr.views.apis.anvil_workspace_api.load_uploaded_file') @@ -574,7 +574,7 @@ def setUp(self): self.mock_datetime = patcher.start() self.mock_datetime.now.side_effect = lambda: datetime(2021, 3, 1, 0, 0, 0) self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.utils.search.add_data_utils.safe_post_to_slack') + patcher = mock.patch('seqr.utils.add_data_utils.safe_post_to_slack') self.mock_slack = patcher.start() self.addCleanup(patcher.stop) diff --git a/seqr/views/apis/auth_api.py b/seqr/views/apis/auth_api.py index 942ec3dbf3..74211df0d6 100644 --- a/seqr/views/apis/auth_api.py +++ b/seqr/views/apis/auth_api.py @@ -1,6 +1,3 @@ -""" -Utility functions related to authentication. -""" from django.contrib.auth import login, authenticate, logout from django.contrib.auth.decorators import login_required from django.contrib.auth.models import User @@ -59,23 +56,14 @@ def logout_view(request): def app_login_required_error(request, exception=None): - """Redirect to login for unhandled 401 error on non-API request""" return redirect(f'{LOGIN_URL}?next={request.path}') def login_required_error(request): - """Returns an HttpResponse with a 401 UNAUTHORIZED error message. - - This is used to redirect AJAX HTTP handlers to the login page. - """ return _unauthorized_error(LOGIN_URL) def policies_required_error(request): - """Returns an HttpResponse with a 401 UNAUTHORIZED error message. - - This is used to redirect AJAX HTTP handlers to the accept policies page. - """ return _unauthorized_error(POLICY_REQUIRED_URL) diff --git a/seqr/views/apis/auth_api_tests.py b/seqr/views/apis/auth_api_tests.py index 9d8f28f288..a448373790 100644 --- a/seqr/views/apis/auth_api_tests.py +++ b/seqr/views/apis/auth_api_tests.py @@ -28,7 +28,7 @@ def test_login_view(self): # send login request without an email req_values = { - 'password': 'not_a_password' + 'password': 'not_a_password' # nosec } response = self.client.post(url, content_type='application/json', data=json.dumps(req_values)) @@ -55,7 +55,7 @@ def test_login_view(self): # send login request with unicode character req_values = { 'email': 'test_new_user@instıtute.com', - 'password': 'password123' + 'password': 'password123' # nosec } response = self.client.post(url, content_type='application/json', data=json.dumps(req_values)) self.assertEqual(response.status_code, 401) @@ -74,7 +74,7 @@ def test_login_view(self): # send login request with a privileged account (data manage or superuser) while Google auth is not enabled req_values = { 'email': 'test_data_manager@test.com', - 'password': 'not-a-password' + 'password': 'not-a-password' # nosec } response = self.client.post(url, content_type='application/json', data=json.dumps(req_values)) diff --git a/seqr/views/apis/awesomebar_api.py b/seqr/views/apis/awesomebar_api.py index 3685f69935..efbb17f05a 100644 --- a/seqr/views/apis/awesomebar_api.py +++ b/seqr/views/apis/awesomebar_api.py @@ -1,4 +1,4 @@ -"""API that generates auto-complete suggestions for the search bar in the header of seqr pages""" +"""API that generates auto-complete suggestions for the search bar in the header of seqr pages.""" from django.db.models import F, Q, Case, When, Value, ExpressionWrapper, BooleanField, CharField from django.db.models.functions import Cast, Coalesce, Concat, Left, Length, NullIf, Replace from django.views.decorators.http import require_GET @@ -119,14 +119,6 @@ def _get_matching_project_groups(query, project_guids): def _get_matching_genes(query, *args): - """Returns genes that match the given query string, and that the user can view. - - Args: - user: Django user - query: String typed into the awesomebar - Returns: - Sorted list of matches where each match is a dictionary of strings - """ result = [] for g in get_queried_genes(query, MAX_RESULTS_PER_CATEGORY): if query.lower() in g['gene_id'].lower(): @@ -147,7 +139,6 @@ def _get_matching_genes(query, *args): def _get_matching_omim(query, *args): - """Returns OMIM records that match the given query string""" records = Omim.objects.filter( Q(phenotype_mim_number__icontains=query) | Q(phenotype_description__icontains=query) ).filter(phenotype_mim_number__isnull=False).annotate( @@ -167,7 +158,6 @@ def _get_matching_omim(query, *args): def _get_matching_hpo_terms(query, *args): - """Returns OMIM records that match the given query string""" records = HumanPhenotypeOntology.objects.filter( Q(hpo_id__icontains=query) | Q(name__icontains=query) ).annotate( @@ -206,8 +196,6 @@ def _get_matching_hpo_terms(query, *args): @login_and_policies_required @require_GET def awesomebar_autocomplete_handler(request): - """Accepts HTTP GET request with q=.. url arg, and returns suggestions""" - query = request.GET.get('q').strip() if not query: return create_json_response({'matches': {}}) diff --git a/seqr/views/apis/case_review_api.py b/seqr/views/apis/case_review_api.py index fd1770f069..2f0687ca08 100644 --- a/seqr/views/apis/case_review_api.py +++ b/seqr/views/apis/case_review_api.py @@ -1,6 +1,3 @@ -""" -APIs used by the case review page -""" import json from django.core.exceptions import PermissionDenied diff --git a/seqr/views/apis/dashboard_api.py b/seqr/views/apis/dashboard_api.py index 1c20f6216b..135f064fd9 100644 --- a/seqr/views/apis/dashboard_api.py +++ b/seqr/views/apis/dashboard_api.py @@ -1,9 +1,7 @@ -""" -APIs used by the main seqr dashboard page -""" from django.db import models +from django.db.models.functions import Coalesce -from seqr.models import ProjectCategory, Sample, RnaSample, Family, Project +from seqr.models import ProjectCategory, Individual, RnaSample, Family, Project from seqr.views.utils.individual_utils import check_project_individuals_deletable from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.orm_to_json_utils import get_json_for_projects @@ -12,15 +10,6 @@ @login_and_policies_required def dashboard_page_data(request): - """Returns a JSON object containing information used by the dashboard page: - :: - - json_response = { - 'user': {..}, - 'familiesByGuid': {..}, - 'individualsByGuid': {..}, - } - """ projects_by_guid = _get_projects_json(request.user) project_categories_by_guid = _retrieve_project_categories_by_guid(projects_by_guid.keys()) @@ -59,13 +48,19 @@ def _get_projects_json(user): projects_by_guid[project_guid]['analysisStatusCounts'] = {} projects_by_guid[project_guid]['analysisStatusCounts'][agg['analysis_status']] = agg['count'] - sample_type_status_counts = _sample_type_counts( - Sample.objects.filter(individual__family__project__in=projects, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) - ) + _sample_type_counts( - RnaSample.objects.filter(individual__family__project__in=projects).annotate(sample_type=models.Value('RNA')) + sample_type_status_counts = list( + Individual.objects.filter(family__project__in=projects).annotate(sample_type=Coalesce( + models.F('active_datasets__sample_type'), models.F('inactive_datasets__sample_type'), + )).filter(sample_type__isnull=False).values('sample_type', project_guid=models.F('family__project__guid')).annotate( + count=models.Count('id', distinct=True) + ) + ) + list( + RnaSample.objects.filter(individual__family__project__in=projects).values( + project_guid=models.F('individual__family__project__guid') + ).annotate(sample_type=models.Value('RNA'), count=models.Count('individual_id', distinct=True)) ) for agg in sample_type_status_counts: - project_guid = agg['individual__family__project__guid'] + project_guid = agg['project_guid'] if 'sampleTypeCounts' not in projects_by_guid[project_guid]: projects_by_guid[project_guid]['sampleTypeCounts'] = {} projects_by_guid[project_guid]['sampleTypeCounts'][agg['sample_type']] = agg['count'] @@ -73,23 +68,7 @@ def _get_projects_json(user): return projects_by_guid -def _sample_type_counts(sample_q): - return list(sample_q.values( - 'individual__family__project__guid', 'sample_type', - ).annotate(count=models.Count('individual_id', distinct=True))) - def _retrieve_project_categories_by_guid(project_guids): - """Retrieves project categories from the database, and returns a 'project_categories_by_guid' dictionary, - while also adding a 'projectCategoryGuids' attribute to each project dict in 'projects_by_guid'. - - Args: - projects_by_guid: Dictionary that maps each project's GUID to a dictionary of key-value - pairs representing attributes of that project. - - Returns: - Dictionary that maps each category's GUID to a dictionary of key-value pairs representing - attributes of that category. - """ if len(project_guids) == 0: return {} diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 6f0fc97ae9..2aa72bd842 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -11,7 +11,7 @@ from clickhouse_search.search import delete_clickhouse_project from seqr.utils.communication_utils import send_project_notification -from seqr.utils.search.add_data_utils import trigger_data_loading, get_missing_family_samples, get_loaded_individual_ids, trigger_delete_families_search +from seqr.utils.add_data_utils import trigger_data_loading, get_missing_family_samples, get_loaded_individual_ids, trigger_delete_families_search from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.vcf_utils import validate_vcf_and_get_samples, get_vcf_list @@ -23,13 +23,14 @@ from seqr.views.utils.permissions_utils import data_manager_required, pm_or_data_manager_required, get_internal_projects from seqr.views.utils.terra_api_utils import anvil_enabled -from seqr.models import Sample, RnaSample, Individual, Project, PhenotypePrioritization +from seqr.models import Dataset, RnaSample, Individual, Project, PhenotypePrioritization from settings import LOADING_DATASETS_DIR, LUIGI_UI_SERVICE_HOSTNAME, LUIGI_UI_SERVICE_PORT logger = SeqrLogger(__name__) RNA = 'RNA' +SEQUENCING_FIELD = 'RNASequencingProduct' TISSUE_FIELD = 'TissueOfOrigin' AIRTABLE_TISSUE_TYPE_MAP = { 'whole_blood': 'Blood', @@ -53,7 +54,7 @@ def update_rna_seq(request): airtable_samples = _get_dataset_type_samples_for_matched_pdos( ['RNA ready to load'], request.user, RNA, None, allow_invalid_pdos=True, sample_fields=[ TISSUE_FIELD, 'WESSampleID_RNAMapping', 'WGSSampleID_RNAMapping', - ], + ], pdo_fields=[SEQUENCING_FIELD], ) sample_metadata_mapping = {} misconfigured_samples = defaultdict(list) @@ -72,10 +73,17 @@ def update_rna_seq(request): error_messages.append('invalid tissue specified') pdos = sample.get('pdos', []) + sequencing_type = None if len(pdos) > 1: error_messages.append('multiple conflicting PDOs') elif not pdos or pdos[0]['project_guid'] is None: error_messages.append('no project specified') + elif not pdos[0][SEQUENCING_FIELD]: + error_messages.append('no sequencing product specified') + else: + sequencing_type = next((st for st, label in RnaSample.SEQUENCING_CHOICES if pdos[0][SEQUENCING_FIELD].startswith(label)), None) + if not sequencing_type: + error_messages.append('invalid sequencing product specified') if error_messages: for sample_id in sample_ids: @@ -83,6 +91,7 @@ def update_rna_seq(request): else: metadata = { 'tissue': TISSUE_TYPE_MAP[sample[TISSUE_FIELD][0]], + 'sequencing_type': sequencing_type, 'project_guid': sample['pdos'][0]['project_guid'], 'sample_id': sample.get('CollaboratorSampleID') or sample['sample_id'], } @@ -219,17 +228,17 @@ def get_loaded_projects(request, genome_version, sample_type, dataset_type): except ValueError as e: return create_json_response({'error': str(e)}, status=400) projects = projects.filter(guid__in=project_samples.keys()) - if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: - exclude_sample_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS + if dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS: + exclude_sample_type = Dataset.SAMPLE_TYPE_WES if sample_type == Dataset.SAMPLE_TYPE_WGS else Dataset.SAMPLE_TYPE_WGS # Include projects with either the matched sample type OR with no loaded data - projects = projects.exclude(family__individual__sample__sample_type=exclude_sample_type) + projects = projects.exclude(family__individual__active_datasets__sample_type=exclude_sample_type) else: # All other data types can only be loaded to projects which already have loaded data - projects = projects.filter(family__individual__sample__sample_type=sample_type) + projects = projects.filter(family__individual__active_datasets__sample_type=sample_type) projects = projects.distinct().order_by('name').values('name', projectGuid=F('guid'), dataTypeLastLoaded=Max( - 'family__individual__sample__loaded_date', - filter=Q(family__individual__sample__dataset_type=dataset_type) & Q(family__individual__sample__sample_type=sample_type), + 'family__individual__active_datasets__loaded_date', + filter=Q(family__individual__active_datasets__dataset_type=dataset_type) & Q(family__individual__active_datasets__sample_type=sample_type), )) if project_samples: @@ -240,10 +249,10 @@ def get_loaded_projects(request, genome_version, sample_type, dataset_type): AIRTABLE_CALLSET_FIELDS = { - (Sample.DATASET_TYPE_MITO_CALLS, Sample.SAMPLE_TYPE_WES): 'MITO_WES_CallsetPath', - (Sample.DATASET_TYPE_MITO_CALLS, Sample.SAMPLE_TYPE_WGS): 'MITO_WGS_CallsetPath', - (Sample.DATASET_TYPE_SV_CALLS, Sample.SAMPLE_TYPE_WES): 'gCNV_CallsetPath', - (Sample.DATASET_TYPE_SV_CALLS, Sample.SAMPLE_TYPE_WGS): 'SV_CallsetPath', + (Dataset.DATASET_TYPE_MITO_CALLS, Dataset.SAMPLE_TYPE_WES): 'MITO_WES_CallsetPath', + (Dataset.DATASET_TYPE_MITO_CALLS, Dataset.SAMPLE_TYPE_WGS): 'MITO_WGS_CallsetPath', + (Dataset.DATASET_TYPE_SV_CALLS, Dataset.SAMPLE_TYPE_WES): 'gCNV_CallsetPath', + (Dataset.DATASET_TYPE_SV_CALLS, Dataset.SAMPLE_TYPE_WGS): 'SV_CallsetPath', } @@ -271,7 +280,7 @@ def load_data(request): request_json = json.loads(request.body) vcf_samples = request_json['vcfSamples'] sample_type = request_json['sampleType'] - dataset_type = request_json.get('datasetType', Sample.DATASET_TYPE_VARIANT_CALLS) + dataset_type = request_json.get('datasetType', Dataset.DATASET_TYPE_VARIANT_CALLS) projects = [json.loads(project) for project in request_json['projects']] project_samples = {p['projectGuid']: p.get('sampleIds') for p in projects} @@ -400,12 +409,17 @@ def trigger_delete_project(request): project_guid = request_json.pop('project') dataset_type = request_json.get('datasetType') project = Project.objects.get(guid=project_guid) - samples = Sample.objects.filter(individual__family__project=project, dataset_type=dataset_type, is_active=True) + datasets = Dataset.objects.filter(active_individuals__family__project=project, dataset_type=dataset_type) sample_types = list( - samples.values_list('sample_type', flat=True).distinct() - ) if dataset_type == Sample.DATASET_TYPE_SV_CALLS else [None] - updated = Sample.bulk_update(user=request.user, update_json={'is_active': False}, queryset=samples) - info = [f'Deactivated search for {len(updated)} individuals'] + datasets.values_list('sample_type', flat=True).distinct() + ) if dataset_type == Dataset.DATASET_TYPE_SV_CALLS else [None] + updated = 0 + for dataset in datasets: + active_individuals = dataset.active_individuals.all() + updated += len(active_individuals) + dataset.inactive_individuals.add(*active_individuals) + dataset.active_individuals.clear() + info = [f'Deactivated search for {updated} individuals'] for sample_type in sample_types: info.append(delete_clickhouse_project(project, dataset_type=dataset_type, sample_type=sample_type)) return create_json_response({'info': info}) @@ -416,7 +430,6 @@ def trigger_delete_family(request): request_json = json.loads(request.body) family_guid = request_json.pop('family') project = Project.objects.get(family__guid=family_guid) - samples = Sample.objects.filter(individual__family__guid=family_guid) info = trigger_delete_families_search(project, [family_guid], request.user) return create_json_response({'info': info}) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 3b688ab47e..b40b2a37d6 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -11,7 +11,7 @@ get_loaded_projects, load_data, trigger_delete_family from seqr.views.utils.orm_to_json_utils import _get_json_for_models from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest -from seqr.models import Individual, Sample, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier, RnaSample, Project, PhenotypePrioritization +from seqr.models import Dataset, Individual, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier, RnaSample, Project, PhenotypePrioritization from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL PROJECT_GUID = 'R0001_1kg' @@ -252,6 +252,7 @@ 'CollaboratorSampleID': 'NA19675_D2', 'SeqrProject': ['https://seqr.broadinstitute.org/project/R0001_1kg/project_page'], 'PDOStatus': ['RNA ready to load'], + 'RNASequencingProduct': ['Tru-Seq polyA 75M'], } }, { @@ -264,6 +265,7 @@ ], 'PDOStatus': ['RNA ready to load', 'Available in seqr'], 'TissueOfOrigin': ['Muscle'], + 'RNASequencingProduct': ['Tru-Seq polyA 75M', 'Kinnex long read total RNA'], } }, { @@ -273,6 +275,7 @@ 'SeqrProject': ['https://seqr.broadinstitute.org/project/R0003_test/project_page'], 'PDOStatus': ['RNA ready to load'], 'TissueOfOrigin': ['Muscle'], + 'RNASequencingProduct': ['Tru-Seq polyA 75M'], } }, { @@ -285,6 +288,7 @@ ], 'PDOStatus': ['RNA ready to load', 'RNA ready to load'], 'TissueOfOrigin': ['Muscle', 'Brain'], + 'RNASequencingProduct': ['Tru-Seq polyA 75M', 'Watchmaker total RNA 200M'], } }, { @@ -295,6 +299,7 @@ 'https://seqr.broadinstitute.org/project/R0002_empty/project_page', ], 'PDOStatus': ['RNA ready to load'], + 'RNASequencingProduct': ['Tru-Seq polyA 75M', 'Watchmaker total RNA 200M'], } }, { @@ -303,10 +308,10 @@ 'CollaboratorSampleID': 'NA12347', 'SeqrProject': [ 'https://seqr.broadinstitute.org/project/R0002_empty/project_page', - 'https://seqr.broadinstitute.org/project/R0004_non_analyst_project/project_page', ], - 'PDOStatus': ['RNA ready to load', 'RNA ready to load'], + 'PDOStatus': ['RNA ready to load'], 'TissueOfOrigin': ['Muscle'], + 'RNASequencingProduct': ['Custom RNA Product'], } }, { @@ -316,6 +321,7 @@ 'SeqrProject': ['R0003_test'], 'PDOStatus': ['RNA ready to load'], 'TissueOfOrigin': ['Custom Tissue'], + 'RNASequencingProduct': ['Custom RNA Product'], } }, { @@ -325,6 +331,7 @@ 'SeqrProject': ['https://seqr.broadinstitute.org/project/R0003_test/project_page'], 'PDOStatus': ['RNA ready to load'], 'TissueOfOrigin': ['Muscle'], + 'RNASequencingProduct': ['Tru-Seq polyA 75M'], } }, *INVALID_AIRTABLE_SAMPLE_RECORDS['records'], @@ -488,6 +495,7 @@ def test_luigi_proxy(self, mock_hostname): ], 'skipped_samples': 'NA19675_D3, NA20878', 'sample_tissue_type': 'Fibroblast', + 'sample_sequencing_type': 'Watchmaker total RNA 200M', 'num_parsed_samples': 4, 'initial_model_count': 7, 'deleted_count': 4, @@ -511,10 +519,11 @@ def _has_expected_file_loading_logs(self, file, file_path, user, info=None, warn self.assert_json_logs(user, expected_logs) - def _check_rna_sample_model(self, individual_id, data_source, data_type, tissue_type, is_active_sample=True): + def _check_rna_sample_model(self, individual_id, data_source, data_type, tissue_type, sequencing_type='T', is_active_sample=True): tissue_type = tissue_type[0] + sequencing_type = sequencing_type[0] rna_samples = RnaSample.objects.filter( - individual_id=individual_id, tissue_type=tissue_type, data_source=data_source, data_type=data_type, + individual_id=individual_id, tissue_type=tissue_type, sequencing_type=sequencing_type, data_source=data_source, data_type=data_type, ) self.assertEqual(len(rna_samples), 1) sample = rna_samples.first() @@ -533,7 +542,7 @@ def test_update_rna_splice_outlier(self, *args, **kwargs): @mock.patch('seqr.views.utils.airtable_utils.BASE_URL', 'https://seqr.broadinstitute.org/') @mock.patch('seqr.utils.communication_utils.BASE_URL', 'https://test-seqr.org/') - @mock.patch('seqr.utils.search.add_data_utils.SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL', 'seqr-data-loading') + @mock.patch('seqr.utils.add_data_utils.SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL', 'seqr-data-loading') @mock.patch('seqr.views.utils.file_utils.tempfile.gettempdir', lambda: 'tmp/') @mock.patch('seqr.views.utils.dataset_utils.os.path.isfile', lambda *args: True) @mock.patch('seqr.utils.communication_utils.send_html_email') @@ -601,12 +610,14 @@ def _set_gzip_file_iter_stdout(mock_file, rows): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'errors': [f'Unable to load the following samples with no match: {loaded_data_row[0]}'], 'warnings': None}) + sequencing_type = params.get('sample_sequencing_type', AIRTABLE_RNA_SAMPLE_RECORDS[0]['fields']['RNASequencingProduct'][0]) airtable_sample_records = [ { **AIRTABLE_RNA_SAMPLE_RECORDS[0], 'fields': { **AIRTABLE_RNA_SAMPLE_RECORDS[0]['fields'], 'TissueOfOrigin': [params['sample_tissue_type']], + 'RNASequencingProduct': [sequencing_type], } }, *AIRTABLE_RNA_SAMPLE_RECORDS[1:], @@ -633,10 +644,12 @@ def _set_gzip_file_iter_stdout(mock_file, rows): self.assertEqual(response.status_code, 400) self.assertListEqual(response.json()['errors'], [ 'Unable to load the following samples from Airtable with no corresponding seqr ID: NA11111', + 'Unable to load the following samples that are improperly configured in Airtable with invalid sequencing product specified: NA12347', 'Unable to load the following samples that are improperly configured in Airtable with invalid tissue specified: NA12348', - 'Unable to load the following samples that are improperly configured in Airtable with multiple conflicting PDOs: NA12345, NA12347', + 'Unable to load the following samples that are improperly configured in Airtable with multiple conflicting PDOs: NA12345', 'Unable to load the following samples that are improperly configured in Airtable with multiple tissues specified: NA12345', 'Unable to load the following samples that are improperly configured in Airtable with no project specified: NA12348', + 'Unable to load the following samples that are improperly configured in Airtable with no sequencing product specified: NA12346', 'Unable to load the following samples that are improperly configured in Airtable with no tissue specified: NA12346', ]) @@ -700,7 +713,7 @@ def _test_basic_data_loading(data, num_parsed_samples, num_loaded_samples, new_s self.assert_expected_airtable_call( call_index=0, filter_formula="AND(LEN({PassingCollaboratorSampleIDs})>0,OR(SEARCH('RNA ready to load',ARRAYJOIN(PDOStatus,';'))))", - fields=['CollaboratorSampleID', 'SeqrCollaboratorSampleID', 'PDOStatus', 'SeqrProject', 'TissueOfOrigin', 'WESSampleID_RNAMapping', 'WGSSampleID_RNAMapping'], + fields=['CollaboratorSampleID', 'SeqrCollaboratorSampleID', 'PDOStatus', 'SeqrProject', 'RNASequencingProduct', 'TissueOfOrigin', 'WESSampleID_RNAMapping', 'WGSSampleID_RNAMapping'], ) return response_json, new_sample_guid @@ -752,7 +765,7 @@ def _test_basic_data_loading(data, num_parsed_samples, num_loaded_samples, new_s # test database models are correct self.assertEqual(model_cls.objects.count(), params['initial_model_count'] - deleted_count) sample_guid = self._check_rna_sample_model(individual_id=1, data_source='new_muscle_samples.tsv.gz', data_type=data_type, - tissue_type=params.get('sample_tissue_type'), is_active_sample=False) + tissue_type=params.get('sample_tissue_type'), sequencing_type=sequencing_type, is_active_sample=False) self.assertSetEqual(set(response_json['sampleGuids']), {sample_guid, new_sample_guid}) # test correct file interactions @@ -1279,8 +1292,7 @@ def test_trigger_delete_family(self): ], }) - family_samples = Sample.objects.filter(individual__family_id=2, is_active=True) - self.assertEqual(family_samples.count(), 0) + self.assertEqual(Dataset.objects.filter(active_individuals__family_id=2).count(), 0) self.assertEqual(len(responses.calls), 1) self.assertDictEqual(json.loads(responses.calls[-1].request.body), { @@ -1299,7 +1311,7 @@ class LocalDataManagerAPITest(AuthenticationTestCase, DataManagerAPITest): PROJECT_OPTION = PROJECT_OPTION WGS_PROJECT_OPTIONS = [EMPTY_PROJECT_OPTION] WES_PROJECT_OPTIONS = [ - {'name': '1kg project nåme with uniçøde', 'projectGuid': 'R0001_1kg', 'dataTypeLastLoaded': '2017-02-05T06:25:55.397Z'}, + {'name': '1kg project nåme with uniçøde', 'projectGuid': 'R0001_1kg', 'dataTypeLastLoaded': '2017-02-05T06:13:55.397Z'}, EMPTY_PROJECT_OPTION, ] PROJECT_OPTIONS = [{'projectGuid': 'R0001_1kg'}, PROJECT_OPTION] @@ -1329,7 +1341,7 @@ def setUp(self): self.mock_subprocess = patcher.start() self.mock_subprocess.side_effect = [self.mock_file_iter] self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.utils.search.add_data_utils.LOADING_DATASETS_DIR', self.TRIGGER_CALLSET_DIR) + patcher = mock.patch('seqr.utils.add_data_utils.LOADING_DATASETS_DIR', self.TRIGGER_CALLSET_DIR) patcher.start() self.addCleanup(patcher.stop) super().setUp() @@ -1449,10 +1461,10 @@ def setUp(self): self.mock_file_iter.wait.return_value = 0 self.mock_subprocess.side_effect = [self.mock_does_file_exist, self.mock_file_iter] self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.utils.search.add_data_utils.safe_post_to_slack') + patcher = mock.patch('seqr.utils.add_data_utils.safe_post_to_slack') self.mock_slack = patcher.start() self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.utils.search.add_data_utils.LOADING_DATASETS_DIR', 'gs://seqr-loading-temp/v3.1') + patcher = mock.patch('seqr.utils.add_data_utils.LOADING_DATASETS_DIR', 'gs://seqr-loading-temp/v3.1') patcher.start() self.addCleanup(patcher.stop) super().setUp() diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index d3254d5203..d3684a5ef6 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -1,6 +1,3 @@ -""" -APIs used to retrieve and modify Individual fields -""" import json from collections import defaultdict from django.contrib.auth.models import User @@ -17,11 +14,11 @@ from seqr.views.utils.json_to_orm_utils import update_family_from_json, update_model_from_json, create_model_from_json from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.note_utils import create_note_handler, update_note_handler, delete_note_handler -from seqr.views.utils.orm_to_json_utils import _get_json_for_model, get_json_for_family_note, get_json_for_samples, \ +from seqr.views.utils.orm_to_json_utils import _get_json_for_model, get_json_for_family_note, get_json_for_datasets, \ get_json_for_matchmaker_submissions, get_json_for_analysis_groups, _get_json_for_families, get_json_for_queryset from seqr.views.utils.project_context_utils import add_families_context, families_discovery_tags, add_project_tag_types, \ MME_TAG_NAME -from seqr.models import Family, FamilyAnalysedBy, Individual, FamilyNote, Sample, VariantTag, AnalysisGroup, RnaSeqTpm, \ +from seqr.models import Family, FamilyAnalysedBy, Individual, FamilyNote, Dataset, VariantTag, AnalysisGroup, RnaSeqTpm, \ PhenotypePrioritization, Project, RnaSample from seqr.views.utils.permissions_utils import check_project_permissions, get_project_and_check_pm_permissions, \ login_and_policies_required, user_is_analyst, has_case_review_permissions, external_anvil_project_can_edit, \ @@ -45,12 +42,12 @@ def family_page_data(request, family_guid): is_analyst = user_is_analyst(request.user) has_case_review_perm = has_case_review_permissions(project, request.user) - sample_models = Sample.objects.filter(individual__family=family) - samples = get_json_for_samples( - sample_models, project_guid=project.guid, family_guid=family_guid, is_analyst=is_analyst - ) + dataset_models = Dataset.objects.filter( + Q(active_individuals__family=family) | Q(inactive_individuals__family=family) + ).distinct() + datasets = get_json_for_datasets(dataset_models, project.guid) response = { - 'samplesByGuid': {s['sampleGuid']: s for s in samples} + 'datasetsByGuid': {d['datasetGuid']: d for d in datasets} } add_families_context(response, families, project.guid, request.user, is_analyst, has_case_review_perm) @@ -63,7 +60,7 @@ def family_page_data(request, family_guid): gene_ids = {gene_id for variant in discovery_variants for gene_id in variant['gene_ids']} discovery_variant_intervals = [dict(zip( ['chrom', 'start', 'end_chrom', 'end', 'svType', 'hasSvType'], - [*get_chrom_pos(v['xpos']), *get_chrom_pos(v['xpos_end']), v['svType'], (v.get('dataset_type') or '').startswith(Sample.DATASET_TYPE_SV_CALLS)] + [*get_chrom_pos(v['xpos']), *get_chrom_pos(v['xpos_end']), v['svType'], (v.get('dataset_type') or '').startswith(Dataset.DATASET_TYPE_SV_CALLS)] )) for v in discovery_variants] omims = Omim.objects.filter( get_omim_intervals_query(discovery_variant_intervals) | Q(gene__gene_id__in=gene_ids) @@ -145,12 +142,6 @@ def family_variant_tag_summary(request, family_guid): @login_and_policies_required def edit_families_handler(request, project_guid): - """Edit or one or more Family records. - - Args: - project_guid (string): GUID of project that contains these individuals. - """ - project = get_project_and_check_pm_permissions(project_guid, request.user) request_json = json.loads(request.body) @@ -201,12 +192,6 @@ def edit_families_handler(request, project_guid): @login_and_policies_required def delete_families_handler(request, project_guid): - """Edit or delete one or more Individual records. - - Args: - project_guid (string): GUID of project that contains these individuals. - """ - project = get_project_and_check_pm_permissions(project_guid, request.user, override_permission_func=external_anvil_project_can_edit) request_json = json.loads(request.body) @@ -238,12 +223,6 @@ def delete_families_handler(request, project_guid): @login_and_policies_required def update_family_fields_handler(request, family_guid): - """Updates the specified field in the Family model. - - Args: - family_guid (string): GUID of the family. - """ - family = Family.objects.get(guid=family_guid) # check permission - can be edited by anyone with access to the project @@ -266,11 +245,6 @@ def _set_display_name(family_json, family_model): @login_and_policies_required def update_family_assigned_analyst(request, family_guid): - """Updates the specified field in the Family model. - - Args: - family_guid (string): GUID of the family. - """ family = Family.objects.get(guid=family_guid) # assigned_analyst can be edited by anyone with access to the project check_project_permissions(family.project, request.user, can_edit=False) @@ -298,13 +272,6 @@ def update_family_assigned_analyst(request, family_guid): @login_and_policies_required def update_family_analysed_by(request, family_guid): - """Updates the specified field in the Family model. - - Args: - family_guid (string): GUID of the family. - field_name (string): Family model field name to update - """ - family = Family.objects.get(guid=family_guid) # analysed_by can be edited by anyone with access to the project check_project_permissions(family.project, request.user, can_edit=False) @@ -319,12 +286,6 @@ def update_family_analysed_by(request, family_guid): @login_and_policies_required def update_family_pedigree_image(request, family_guid): - """Updates the specified field in the Family model. - - Args: - family_guid (string): GUID of the family. - """ - family = Family.objects.get(guid=family_guid) # check permission @@ -388,6 +349,7 @@ def receive_families_table_handler(request, project_guid): Args: request (object): Django request object project_guid (string): project GUID + """ project = get_project_and_check_pm_permissions(project_guid, request.user) @@ -488,24 +450,25 @@ def get_family_rna_seq_data(request, family_guid, gene_id): family = Family.objects.get(guid=family_guid) check_project_permissions(family.project, request.user) - response = defaultdict(lambda: {'individualData': {}}) + response = defaultdict(lambda: defaultdict(lambda: {'individualData': {}})) tpm_data = RnaSeqTpm.objects.filter( gene_id=gene_id, sample__individual__family=family).prefetch_related('sample', 'sample__individual') for tpm in tpm_data: indiv = tpm.sample.individual - response[tpm.sample.tissue_type]['individualData'][indiv.display_name or indiv.individual_id] = tpm.tpm + response[tpm.sample.tissue_type][tpm.sample.sequencing_type]['individualData'][indiv.display_name or indiv.individual_id] = tpm.tpm project_guids = get_project_guids_user_can_view(request.user) internal_projects = get_internal_projects() if anvil_enabled() else None for tissue in response.keys(): - tpms = RnaSeqTpm.objects.filter(sample__tissue_type=tissue, gene_id=gene_id) - response[tissue]['myData'] = list(tpms.filter( - sample__individual__family__project__guid__in=project_guids, - ).order_by('tpm').values_list('tpm', flat=True)) - if internal_projects is not None: - response[tissue]['rdgData'] = list(tpms.filter( - sample__individual__family__project__in=internal_projects, + for sequencing_type in response[tissue].keys(): + tpms = RnaSeqTpm.objects.filter(sample__tissue_type=tissue, sample__sequencing_type=sequencing_type, gene_id=gene_id) + response[tissue][sequencing_type]['myData'] = list(tpms.filter( + sample__individual__family__project__guid__in=project_guids, ).order_by('tpm').values_list('tpm', flat=True)) + if internal_projects is not None: + response[tissue][sequencing_type]['rdgData'] = list(tpms.filter( + sample__individual__family__project__in=internal_projects, + ).order_by('tpm').values_list('tpm', flat=True)) return create_json_response(response) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index 67fc881917..6a52e6eaec 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- import json + import mock from copy import deepcopy from datetime import datetime from django.core.files.uploadedfile import SimpleUploadedFile -from django.db import connections from django.urls.base import reverse from matchmaker.models import MatchmakerSubmission @@ -14,9 +14,9 @@ family_variant_tag_summary, update_family_analysis_groups, get_family_rna_seq_data, get_family_phenotype_gene_scores from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, \ FAMILY_NOTE_FIELDS, FAMILY_FIELDS, IGV_SAMPLE_FIELDS, \ - SAMPLE_FIELDS, INDIVIDUAL_FIELDS, INTERNAL_INDIVIDUAL_FIELDS, INTERNAL_FAMILY_FIELDS, CASE_REVIEW_FAMILY_FIELDS, \ + DATASET_FIELDS, INDIVIDUAL_FIELDS, INTERNAL_INDIVIDUAL_FIELDS, INTERNAL_FAMILY_FIELDS, CASE_REVIEW_FAMILY_FIELDS, \ MATCHMAKER_SUBMISSION_FIELDS, TAG_TYPE_FIELDS, CASE_REVIEW_INDIVIDUAL_FIELDS -from seqr.models import FamilyAnalysedBy, AnalysisGroup, Sample +from seqr.models import FamilyAnalysedBy, AnalysisGroup, Individual FAMILY_GUID = 'F000001_1' FAMILY_GUID2 = 'F000002_2' @@ -48,7 +48,7 @@ def test_family_page_data(self): response_json = response.json() response_keys = { - 'familiesByGuid', 'individualsByGuid', 'familyNotesByGuid', 'samplesByGuid', 'igvSamplesByGuid', + 'familiesByGuid', 'individualsByGuid', 'familyNotesByGuid', 'datasetsByGuid', 'igvSamplesByGuid', 'mmeSubmissionsByGuid', } self.assertSetEqual(set(response_json.keys()), response_keys) @@ -73,7 +73,7 @@ def test_family_page_data(self): self.assertEqual(len(response_json['individualsByGuid']), 3) individual = response_json['individualsByGuid'][INDIVIDUAL_GUID] - individual_fields = {'sampleGuids', 'igvSampleGuids', 'mmeSubmissionGuid', 'phenotypePrioritizationTools', 'rnaSample'} + individual_fields = {'igvSampleGuids', 'mmeSubmissionGuid', 'phenotypePrioritizationTools', 'rnaSample'} individual_fields.update(INDIVIDUAL_FIELDS) self.assertSetEqual(set(individual.keys()), individual_fields) self.assertListEqual([ @@ -95,12 +95,15 @@ def test_family_page_data(self): self.assertSetEqual({PROJECT_GUID}, {i['projectGuid'] for i in response_json['individualsByGuid'].values()}) self.assertSetEqual({FAMILY_GUID}, {i['familyGuid'] for i in response_json['individualsByGuid'].values()}) - self.assertEqual(len(response_json['samplesByGuid']), 3) - self.assertSetEqual(set(next(iter(response_json['samplesByGuid'].values())).keys()), SAMPLE_FIELDS) - self.assertSetEqual({PROJECT_GUID}, {s['projectGuid'] for s in response_json['samplesByGuid'].values()}) - self.assertSetEqual({FAMILY_GUID}, {s['familyGuid'] for s in response_json['samplesByGuid'].values()}) - self.assertEqual(len(individual['sampleGuids']), 1) - self.assertTrue(set(individual['sampleGuids']).issubset(set(response_json['samplesByGuid'].keys()))) + self.assertEqual(len(response_json['datasetsByGuid']), 2) + dataset = response_json['datasetsByGuid']['S000129_na19675'] + self.assertSetEqual(set(dataset.keys()), DATASET_FIELDS) + self.assertSetEqual({PROJECT_GUID}, {s['projectGuid'] for s in response_json['datasetsByGuid'].values()}) + self.assertListEqual( + dataset['activeIndividuals'], + ['I000001_na19675', 'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000007_na20870', 'I000009_na20874'], + ) + self.assertListEqual(dataset['inactiveIndividuals'], ['I000003_na19679']) self.assertEqual(len(response_json['igvSamplesByGuid']), 1) self.assertSetEqual(set(next(iter(response_json['igvSamplesByGuid'].values())).keys()), IGV_SAMPLE_FIELDS) @@ -209,7 +212,7 @@ def test_family_page_data(self): self.assertEqual(family['projectGuid'], PROJECT_GUID) self.assertSetEqual(set(family['individualGuids']), set(response_json['individualsByGuid'].keys())) self.assertEqual(len(response_json['individualsByGuid']), 3) - self.assertEqual(len(response_json['samplesByGuid']), 7) + self.assertEqual(len(response_json['datasetsByGuid']), 3) self.assertEqual(len(response_json['igvSamplesByGuid']), 0) self.assertEqual(len(response_json['mmeSubmissionsByGuid']), 0) self.assertEqual(len(response_json['familyNotesByGuid']), 0) @@ -324,7 +327,7 @@ def test_delete_families_handler(self, mock_pm_group, mock_internal_namespaces): # Test success MatchmakerSubmission.objects.update(deleted_date=datetime.now()) - Sample.objects.update(is_active=False) + Individual.active_datasets.through.objects.all().delete() response = self.client.post(url, content_type='application/json', data=json.dumps(req_values)) self.assertEqual(response.status_code, 200) @@ -645,17 +648,17 @@ def test_get_family_rna_seq_data(self): self.assertEqual(response.status_code, 200) expected_response = { - 'F': {'individualData': {'NA19675_1': 1.01}, 'myData': [1.01]}, - 'M': {'individualData': {'NA19675_1': 8.38}, 'myData': [8.38]} + 'F': {'W': {'individualData': {'NA19675_1': 1.01}, 'myData': [1.01]}}, + 'M': {'T': {'individualData': {'NA19675_1': 8.38}, 'myData': [8.38]}} } if self.INCLUDE_RDG_TPMS: - expected_response = {k: {**v, 'rdgData': [*v['myData']]} for k, v in expected_response.items()} + expected_response = {tissue: {k: {**v, 'rdgData': [*v['myData']]} for k, v in data.items()} for tissue, data in expected_response.items()} self.assertDictEqual(response.json(), expected_response) self.login_manager() response = self.client.get(url) self.assertEqual(response.status_code, 200) - expected_response['F']['myData'].append(2.36) + expected_response['F']['W']['myData'].append(2.36) self.assertDictEqual(response.json(), expected_response) def test_get_family_phenotype_gene_scores(self): diff --git a/seqr/views/apis/feature_updates_api.py b/seqr/views/apis/feature_updates_api.py index 140b650182..e8e7940340 100644 --- a/seqr/views/apis/feature_updates_api.py +++ b/seqr/views/apis/feature_updates_api.py @@ -19,10 +19,10 @@ def fetch_feature_updates(): def get_feature_updates(request): - """ - Fetches the feature-updates GitHub Discussion Atom feed, converts feed entries into markdown, and returns + """Fetches the feature-updates GitHub Discussion Atom feed, converts feed entries into markdown, and returns markdown and information for each feed entry. """ + entries = [] for entry in fetch_feature_updates(): # Atom feeds can have multiple content elements per feed entry diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 94cb1c80fa..dc0e8f248b 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -12,7 +12,7 @@ from io import BytesIO from openpyxl import load_workbook -from seqr.models import Individual, Sample, SavedVariant, VariantTag +from seqr.models import Individual, Dataset, SavedVariant, VariantTag from seqr.views.apis.individual_api import edit_individuals_handler, update_individual_handler, \ delete_individuals_handler, receive_individuals_table_handler, save_individuals_table_handler, \ receive_individuals_metadata_handler, save_individuals_metadata_table_handler, update_individual_hpo_terms, \ @@ -206,7 +206,9 @@ def test_edit_individuals(self, mock_pm_group): ]) # send valid request - Sample.objects.filter(guid__in=['S000130_na19678', 'S000135_na20870']).update(is_active=False) + for dataset in Dataset.objects.filter(guid__in=['S000130_na19678', 'S000129_na19675']): + dataset.inactive_individuals.set(dataset.active_individuals.all()) + dataset.active_individuals.set(set()) response = self.client.post(edit_individuals_url, content_type='application/json', data=json.dumps({ 'individuals': [INDIVIDUAL_IDS_UPDATE_DATA, INDIVIDUAL_FAMILY_UPDATE_DATA] })) @@ -345,6 +347,7 @@ def test_edit_individuals(self, mock_pm_group): 'updateFields': ['mother'], 'updateType': 'update', }}), + ('Reloading dictionary seqrdb_affected_status_dict', None), ('Triggering rebuild_gt_stats for R0004_non_analyst_project', None), (f'Error Triggering Rebuild Gt Stats: 400 Client Error: Bad Request for url: {TRIGGER_RELOAD_URL}', { 'severity': 'ERROR', @@ -369,7 +372,9 @@ def test_delete_individuals(self, mock_pm_group): })) self.assertEqual(response.status_code, 400) self.assertListEqual(response.json()['errors'], ['Unable to delete individuals with active search sample: NA19678']) - Sample.objects.filter(guid__in=['S000130_na19678', 'S000143_na20885', 'S000173_na21987']).update(is_active=False) + for dataset in Dataset.objects.filter(guid__in=['S000130_na19678', 'S000143_na20885', 'S000147_na21234']): + dataset.inactive_individuals.set(dataset.active_individuals.all()) + dataset.active_individuals.set(set()) # send valid requests response = self.client.post(individuals_url, content_type='application/json', data=json.dumps({ @@ -385,6 +390,7 @@ def test_delete_individuals(self, mock_pm_group): }) self.assertFalse('I000002_na19678' in response_json['familiesByGuid']['F000001_1']['individualGuids']) self.assertIsNone(response_json['familiesByGuid']['F000001_1']['pedigreeImage']) + self.assertFalse(Dataset.objects.filter(guid='S000130_na19678').exists()) # Test PM permission pm_required_delete_individuals_url = reverse(delete_individuals_handler, args=[PM_REQUIRED_PROJECT_GUID]) @@ -433,6 +439,11 @@ def test_delete_individuals(self, mock_pm_group): 'individuals': [EXTERNAL_WORKSPACE_INDIVIDUAL_UPDATE_DATA] })) self.assertEqual(response.status_code, 200 if self.HAS_EXTERNAL_PROJECT_ACCESS else 403) + if self.HAS_EXTERNAL_PROJECT_ACCESS: + self.assertListEqual( + list(Dataset.objects.get(guid='S000147_na21234').inactive_individuals.order_by('id').values_list('guid', flat=True)), + ['I000018_na21234', 'I000021_na21654'], + ) def test_individuals_table_handler_errors(self): individuals_url = reverse(receive_individuals_table_handler, args=[PROJECT_GUID]) @@ -1364,16 +1375,16 @@ def test_get_individual_rna_seq_data(self): 'ENSG00000135953': [{ 'geneId': 'ENSG00000135953', 'zScore': 7.31, 'pValue': 0.00000000000948, 'pAdjust': 0.00000000781, 'isSignificant': True, - 'tissueType': 'M', + 'tissueType': 'M', 'sequencingType': 'T', }], 'ENSG00000240361': [{ 'geneId': 'ENSG00000240361', 'zScore': -4.08, 'pValue': 5.88, 'pAdjust': 0.09, 'isSignificant': False, - 'tissueType': 'M', + 'tissueType': 'M', 'sequencingType': 'T', }], 'ENSG00000268903': [{ 'geneId': 'ENSG00000268903', 'zScore': 7.08, 'pValue':0.000000000588, 'pAdjust': 0.00000000139, 'isSignificant': True, - 'tissueType': 'M', + 'tissueType': 'M', 'sequencingType': 'T', }], }, 'spliceOutliers': { @@ -1388,7 +1399,7 @@ def test_get_individual_rna_seq_data(self): 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 3.08e-56, 'pValue': 1.08e-56, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, 'start': 132885746, 'strand': '*', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34, - 'tissueType': 'F', + 'tissueType': 'F', 'sequencingType': 'W', }, outliers_by_pos[132885746] ) diff --git a/seqr/views/apis/project_api.py b/seqr/views/apis/project_api.py index b5793f8db7..db0eb69245 100644 --- a/seqr/views/apis/project_api.py +++ b/seqr/views/apis/project_api.py @@ -1,7 +1,3 @@ -""" -APIs for updating project metadata, as well as creating or deleting projects -""" - import json from collections import defaultdict from django.contrib.postgres.aggregates import ArrayAgg @@ -12,7 +8,7 @@ from notifications.models import Notification from matchmaker.models import MatchmakerSubmission -from seqr.models import Project, Family, Individual, Sample, RnaSample, FamilyNote, PhenotypePrioritization, CAN_EDIT +from seqr.models import Project, Family, Individual, Dataset, RnaSample, FamilyNote, PhenotypePrioritization, CAN_EDIT from seqr.utils.file_utils import file_iter from seqr.utils.logging_utils import SeqrLogger from seqr.views.utils.airtable_utils import AirtableSession, ANVIL_REQUEST_TRACKING_TABLE @@ -21,7 +17,7 @@ from seqr.views.utils.individual_utils import delete_individuals from seqr.views.utils.json_utils import create_json_response, _to_snake_case, _to_camel_case from seqr.views.utils.json_to_orm_utils import update_project_from_json, create_model_from_json, update_model_from_json -from seqr.views.utils.orm_to_json_utils import _get_json_for_project, get_json_for_samples, \ +from seqr.views.utils.orm_to_json_utils import _get_json_for_project, get_json_for_datasets, \ get_json_for_project_collaborator_list, get_json_for_matchmaker_submissions, \ get_json_for_family_notes, _get_json_for_individuals, get_json_for_project_collaborator_groups, \ FAMILY_ADDITIONAL_VALUES @@ -39,19 +35,6 @@ @pm_required def create_project_handler(request): - """Create a new project. - - HTTP POST - Request body - should contain json params: - name: Project name - description: Project description - - Response body - will be json with the following structure, representing the ,created project: - { - 'projectsByGuid': { : { ... ... } } - } - - """ request_json = json.loads(request.body) required_fields = ['name', 'genomeVersion'] @@ -92,29 +75,6 @@ def _is_valid_anvil_workspace(request_json, user): @login_and_policies_required def update_project_handler(request, project_guid): - """Update project metadata - including one or more of these fields: name, description - - Args: - project_guid (string): GUID of the project that should be updated - - HTTP POST - Request body - should contain the following json structure: - { - 'form' : { - 'name': , - 'description': , - } - } - - Response body - will contain the following structure, representing the updated project: - { - 'projectsByGuid': { - : { ... ... } - } - } - - """ - project = Project.objects.get(guid=project_guid) check_project_permissions(project, request.user, can_edit=True) @@ -156,12 +116,6 @@ def update_project_workspace(request, project_guid): @login_and_policies_required def delete_project_handler(request, project_guid): - """Delete project - request handler. - - Args: - project_guid (string): GUID of the project to delete - """ - _delete_project(project_guid, request.user) return create_json_response({ @@ -173,12 +127,6 @@ def delete_project_handler(request, project_guid): @login_and_policies_required def project_page_data(request, project_guid): - """ - Returns a JSON object containing basic project information - - Args: - project_guid (string): GUID of the Project to retrieve data for. - """ project = get_project_and_check_permissions(project_guid, request.user) update_project_from_json(project, {'last_accessed_date': timezone.now()}, request.user) return create_json_response({ @@ -265,25 +213,24 @@ def project_families(request, project_guid): def project_overview(request, project_guid): project = get_project_and_check_permissions(project_guid, request.user) - sample_load_counts, sample_models = _sample_load_counts( - Sample, project, 'sample_type', 'dataset_type', loadedDate=TruncDate('loaded_date'), - ) - rna_sample_load_counts, _ = _sample_load_counts( - RnaSample, project, sample_type=Value('RNA'), dataset_type=F('data_type'), loadedDate=TruncDate('created_date'), - ) - - first_loaded_samples = sample_models.order_by('individual__family', 'loaded_date').distinct('individual__family').values_list('id', flat=True) - samples = sample_models.filter(Q(is_active=True) | Q(id__in=first_loaded_samples)) - samples_by_guid = {s['sampleGuid']: s for s in get_json_for_samples(samples, project_guid=project_guid)} + datasets = Dataset.objects.filter( + Q(active_individuals__family__project=project) | Q(inactive_individuals__family__project=project) + ).distinct() + datasets_by_guid = {d['datasetGuid']: d for d in get_json_for_datasets(datasets, project_guid)} + rna_sample_load_counts = RnaSample.objects.filter( + individual__family__project=project, + ).values('data_type', loadedDate=TruncDate('created_date')).order_by('loadedDate').annotate( + familyCounts=ArrayAgg('individual__family__guid'), + ) grouped_sample_counts = defaultdict(list) - for s in sample_load_counts + rna_sample_load_counts: + for s in rna_sample_load_counts: s['familyCounts'] = {f: s['familyCounts'].count(f) for f in s['familyCounts']} - grouped_sample_counts[f'{s.pop("sample_type")}__{s.pop("dataset_type")}'].append(s) + grouped_sample_counts[s.pop('data_type')].append(s) - project_json = {'projectGuid': project_guid, 'sampleCounts': grouped_sample_counts} + project_json = {'projectGuid': project_guid, 'rnaSampleCounts': grouped_sample_counts} response = { - 'samplesByGuid': samples_by_guid, + 'datasetsByGuid': datasets_by_guid, } add_project_tag_type_counts(project, response, project_json=project_json) @@ -299,13 +246,6 @@ def project_overview(request, project_guid): return create_json_response(response) -def _sample_load_counts(sample_cls, project, *args, **kwargs): - sample_models = sample_cls.objects.filter(individual__family__project=project) - return list(sample_models.values(*args, **kwargs).order_by('loadedDate').annotate( - familyCounts=ArrayAgg('individual__family__guid')) - ), sample_models - - @login_and_policies_required def project_collaborators(request, project_guid): project = get_project_and_check_permissions(project_guid, request.user) @@ -330,16 +270,6 @@ def project_individuals(request, project_guid): }) -@login_and_policies_required -def project_samples(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user) - samples = Sample.objects.filter(individual__family__project=project) - - return create_json_response({ - 'samplesByGuid': {s['sampleGuid']: s for s in get_json_for_samples(samples, project_guid=project_guid)}, - }) - - @login_and_policies_required def project_analysis_groups(request, project_guid): project = get_project_and_check_permissions(project_guid, request.user) @@ -439,18 +369,10 @@ def subscribe_project_notifications(request, project_guid): def _delete_project(project_guid, user): - """Delete project. - - Args: - project_guid (string): GUID of the project to delete - user (object): Django ORM model for the user - """ project = Project.objects.get(guid=project_guid) check_user_created_object_permissions(project, user) - individual_guids_to_delete = Individual.objects.filter( - family__project__guid=project_guid).values_list('guid', flat=True) - delete_individuals(project, individual_guids_to_delete, user) + delete_individuals(project, individual_guids=None, user=user) Family.bulk_delete(user, project=project) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index 43b4fcf6ad..16ce8a0629 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -3,7 +3,6 @@ from copy import deepcopy from datetime import datetime from django.contrib.auth.models import Group -from django.db import connections from django.urls.base import reverse import responses @@ -12,7 +11,7 @@ from seqr.views.apis.project_api import create_project_handler, delete_project_handler, update_project_handler, \ project_page_data, project_families, project_overview, project_mme_submisssions, project_individuals, \ project_analysis_groups, update_project_workspace, project_family_notes, project_collaborators, project_locus_lists, \ - project_samples, project_notifications, mark_read_project_notifications, subscribe_project_notifications, \ + project_notifications, mark_read_project_notifications, subscribe_project_notifications, \ update_project_rna_seq, load_rna_seq_sample_data from seqr.views.apis.data_manager_api_tests import RNA_OUTLIER_SAMPLE_DATA, RNA_OUTLIER_MUSCLE_SAMPLE_GUID, RNA_TPM_SAMPLE_DATA, \ RNA_TPM_MUSCLE_SAMPLE_GUID, RNA_SPLICE_SAMPLE_DATA, RNA_SPLICE_SAMPLE_GUID, PLACEHOLDER_GUID, \ @@ -20,7 +19,7 @@ from seqr.views.utils.terra_api_utils import TerraAPIException, TerraRefreshTokenFailedException from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, \ PROJECT_FIELDS, LOCUS_LIST_FIELDS, PA_LOCUS_LIST_FIELDS, NO_INTERNAL_CASE_REVIEW_INDIVIDUAL_FIELDS, \ - SAMPLE_FIELDS, SUMMARY_FAMILY_FIELDS, INTERNAL_INDIVIDUAL_FIELDS, INDIVIDUAL_FIELDS, TAG_TYPE_FIELDS, \ + DATASET_FIELDS, SUMMARY_FAMILY_FIELDS, INTERNAL_INDIVIDUAL_FIELDS, INDIVIDUAL_FIELDS, TAG_TYPE_FIELDS, \ FAMILY_NOTE_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, ANALYSIS_GROUP_FIELDS, \ EXT_WORKSPACE_NAMESPACE, TEST_EMPTY_PROJECT_WORKSPACE, DYNAMIC_ANALYSIS_GROUP_FIELDS @@ -341,12 +340,12 @@ def test_project_overview(self): response_json = response.json() response_keys = { - 'projectsByGuid', 'samplesByGuid', 'familyTagTypeCounts', + 'projectsByGuid', 'datasetsByGuid', 'familyTagTypeCounts', } self.assertSetEqual(set(response_json.keys()), response_keys) project_fields = { - 'variantTagTypes', 'variantFunctionalTagTypes', 'sampleCounts', + 'variantTagTypes', 'variantFunctionalTagTypes', 'rnaSampleCounts', 'projectGuid', 'mmeDeletedSubmissionCount', 'mmeSubmissionCount', } project_response = response_json['projectsByGuid'][PROJECT_GUID] @@ -374,25 +373,16 @@ def test_project_overview(self): 'order': 99, 'numTags': 1, }) - self.assertDictEqual(project_response['sampleCounts'], { - 'WES__SNV_INDEL': [{ - 'familyCounts': { - 'F000001_1': 3, 'F000002_2': 3, 'F000003_3': 1, 'F000004_4': 1, 'F000005_5': 1, 'F000006_6': 1, - 'F000007_7': 1, 'F000008_8': 1, 'F000010_10': 1, - }, - 'loadedDate': '2017-02-05', - }], - 'WES__SV': [{'familyCounts': {'F000002_2': 3}, 'loadedDate': '2018-02-05'}], - 'WES__MITO': [{'familyCounts': {'F000002_2': 1}, 'loadedDate': '2022-02-05'}], - 'RNA__S': [{'familyCounts': {'F000001_1': 2}, 'loadedDate': '2017-02-05'}], - 'RNA__T': [{'familyCounts': {'F000001_1': 2}, 'loadedDate': '2017-02-05'}], - 'RNA__E': [{'familyCounts': {'F000001_1': 1}, 'loadedDate': '2017-02-05'}], + self.assertDictEqual(project_response['rnaSampleCounts'], { + 'S': [{'familyCounts': {'F000001_1': 2}, 'loadedDate': '2017-02-05'}], + 'T': [{'familyCounts': {'F000001_1': 2}, 'loadedDate': '2017-02-05'}], + 'E': [{'familyCounts': {'F000001_1': 1}, 'loadedDate': '2017-02-05'}], }) self.assertEqual(project_response['mmeSubmissionCount'], 1) self.assertEqual(project_response['mmeDeletedSubmissionCount'], 0) - self.assertEqual(len(response_json['samplesByGuid']), 16) - self.assertSetEqual(set(next(iter(response_json['samplesByGuid'].values())).keys()), SAMPLE_FIELDS) + self.assertEqual(len(response_json['datasetsByGuid']), 5) + self.assertSetEqual(set(next(iter(response_json['datasetsByGuid'].values())).keys()), DATASET_FIELDS) self.assertDictEqual(response_json['familyTagTypeCounts'], { 'F000001_1': {'Review': 1, 'Tier 1 - Novel gene and phenotype': 1, 'MME Submission': 1}, 'F000002_2': {'AIP': 1, 'Excluded': 1, 'Known gene for phenotype': 1}, @@ -543,24 +533,6 @@ def test_project_individuals(self): NO_INTERNAL_CASE_REVIEW_INDIVIDUAL_FIELDS, ) - def test_project_samples(self): - url = reverse(project_samples, args=[PROJECT_GUID]) - self.check_collaborator_login(url) - - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - - response_json = response.json() - response_keys = {'samplesByGuid'} - self.assertSetEqual(set(response_json.keys()), response_keys) - - self.assertEqual(len(response_json['samplesByGuid']), 17) - self.assertSetEqual(set(next(iter(response_json['samplesByGuid'].values())).keys()), SAMPLE_FIELDS) - - # Test empty project - empty_url = reverse(project_samples, args=[EMPTY_PROJECT_GUID]) - self._check_empty_project(empty_url, response_keys) - def test_project_analysis_groups(self): url = reverse(project_analysis_groups, args=[PROJECT_GUID]) self.check_collaborator_login(url) @@ -731,7 +703,7 @@ def test_update_project_rna_tpm(self): self._test_update_project_rna('T', **RNA_DATA_TYPE_PARAMS['T'], single_sample_file=True) def test_update_project_rna_splice_outlier(self): - self._test_update_project_rna('S', **RNA_DATA_TYPE_PARAMS['S'], tissue='F', allow_missing_gene=True) + self._test_update_project_rna('S', **RNA_DATA_TYPE_PARAMS['S'], tissue='F', sequencing_type='W', allow_missing_gene=True) @mock.patch('seqr.utils.communication_utils.BASE_URL', 'https://test-seqr.org/') @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') @@ -746,7 +718,7 @@ def test_update_project_rna_splice_outlier(self): def _test_update_project_rna(self, data_type, mock_subprocess, mock_does_file_exist, mock_open, mock_os, mock_send_slack, mock_send_email, mock_datetime, sample_guid=None, model_cls=None, rows=None, parsed_file_data=None, required_columns=None, allow_missing_gene=False, - tissue='M', message_data_type=None, single_sample_file=False, **kwargs): + tissue='M', sequencing_type='T', message_data_type=None, single_sample_file=False, **kwargs): mock_datetime.now.return_value = datetime(2025, 4, 15) mock_os.path.join.side_effect = lambda *args: '/'.join(args) initial_model_count = model_cls.objects.count() @@ -758,7 +730,7 @@ def _test_update_project_rna(self, data_type, mock_subprocess, mock_does_file_ex # Test errors file = f'{self.TEMP_DIR}/new_samples.tsv.gz' - body = {'dataType': data_type, 'file': file, 'tissue': tissue} + body = {'dataType': data_type, 'file': file, 'tissue': tissue, 'sequencingType': sequencing_type} self._set_file_not_found(file, mock_subprocess, mock_does_file_exist, mock_open) self.reset_logs() response = self.client.post(url, content_type='application/json', data=json.dumps(body)) @@ -813,7 +785,7 @@ def _test_update_project_rna(self, data_type, mock_subprocess, mock_does_file_ex # test database models are correct self.assertEqual(model_cls.objects.count(), initial_model_count - initial_sample_model_count) rna_samples = RnaSample.objects.filter( - tissue_type=tissue, data_type=data_type, data_source='new_samples.tsv.gz', is_active=False, + tissue_type=tissue, data_type=data_type, data_source='new_samples.tsv.gz', is_active=False, sequencing_type=sequencing_type, ) self.assertEqual(rna_samples.count(), 1 if single_sample_file else 2) guid_map = {'NA19675_1': rna_samples.get(individual_id=1).guid} diff --git a/seqr/views/apis/project_categories_api.py b/seqr/views/apis/project_categories_api.py index 976bf2bee1..55f15ea3c2 100644 --- a/seqr/views/apis/project_categories_api.py +++ b/seqr/views/apis/project_categories_api.py @@ -1,4 +1,3 @@ -"""APIs for setting Project categories""" import json from seqr.models import Project, ProjectCategory @@ -10,31 +9,6 @@ @login_and_policies_required def update_project_categories_handler(request, project_guid): - """Update ProjectCategories for the given project. - - Args: - project_guid (string): GUID of the project that should be updated - - HTTP POST - Request body - should contain the following json structure: - { - 'form' : { - 'categories': a list of category GUIDs for the categories assigned to the given project - } - } - - Response body - will contain the following structure, representing the updated project, - as well all categories in seqr: - { - 'projectsByGuid': { - : { ... ... } - } - 'projectCategoriesByGuid': { - : { ... ... } - : { ... ... } - } - } - """ project = Project.objects.get(guid=project_guid) # check permissions @@ -58,14 +32,6 @@ def update_project_categories_handler(request, project_guid): def _update_project_categories(project, user, category_guids): - """Updates the stored categories for the given project. - - Args: - project (project): Django Project model - user (User): Django User model - category_guids (set): set of category GUIDs to apply to the given project - """ - category_guids = set(category_guids) project_categories_by_guid = {} # keep track of new and removed categories so client can be updated. diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index 165cd931bc..3d2589a250 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -24,7 +24,7 @@ from seqr.views.utils.terra_api_utils import anvil_enabled from seqr.views.utils.variant_utils import DISCOVERY_CATEGORY -from seqr.models import Project, Family, FamilyAnalysedBy, Sample, RnaSample, Individual +from seqr.models import Project, Family, FamilyAnalysedBy, Dataset, RnaSample, Individual from settings import GREGOR_DATA_MODEL_URL @@ -60,10 +60,13 @@ def seqr_stats(request): grouped_sample_counts = defaultdict(dict) for project_key, projects in project_models.items(): - samples_counts = _get_sample_counts(Sample.objects.filter(individual__family__project__in=projects)) + samples_counts = _get_sample_counts( + Dataset.objects.filter(active_individuals__family__project__in=projects), + count=Count('active_individuals'), + ) samples_counts.update(_get_sample_counts( - RnaSample.objects.filter(individual__family__project__in=projects).annotate(sample_type=Value('RNA')), - data_type_key='data_type') + RnaSample.objects.filter(individual__family__project__in=projects, is_active=True).annotate(sample_type=Value('RNA')), + data_type_key='data_type', count=Count('*')) ) for k, v in samples_counts.items(): grouped_sample_counts[k][project_key] = v @@ -80,8 +83,8 @@ def seqr_stats(request): }) -def _get_sample_counts(sample_q, data_type_key='dataset_type'): - samples_agg = sample_q.filter(is_active=True).values('sample_type', data_type_key).annotate(count=Count('*')) +def _get_sample_counts(sample_q, count=None, data_type_key='dataset_type'): + samples_agg = sample_q.values('sample_type', data_type_key).annotate(count=count) return { f'{sample_agg["sample_type"]}__{sample_agg[data_type_key]}': sample_agg['count'] for sample_agg in samples_agg } @@ -176,8 +179,8 @@ def _add_row(row, family_id, row_type): 'congenital_status': Individual.ONSET_AGE_LOOKUP[individual.onset_age] if individual.onset_age else 'Unknown', **anvil_export_airtable_fields(airtable_metadata, has_dbgap_submission), }, - get_additional_sample_fields=lambda sample, *args: { - 'entity:sample_id': sample.individual.individual_id, + get_additional_sample_fields=lambda individual, *args: { + 'entity:sample_id': individual.individual_id, 'sequencing_center': 'Broad', }, family_fields={'phenotype_group': { @@ -379,11 +382,11 @@ def gregor_export(request): grouped_data_type_individuals = _get_individual_data_types(projects) # If multiple individual records, prefer WGS - individual_lookup = { + individual_lookup = [ next(data_type_individuals[data_type.upper()] for data_type in GREGOR_DATA_TYPES - if data_type_individuals.get(data_type.upper())): None + if data_type_individuals.get(data_type.upper())) for data_type_individuals in grouped_data_type_individuals.values() - } + ] participant_rows = [] family_map = {} @@ -402,7 +405,7 @@ def _add_row(row, family_id, row_type): parse_anvil_metadata( projects, user=request.user, - individual_samples=individual_lookup, + individuals=individual_lookup, individual_data_types=grouped_data_type_individuals, add_row=_add_row, format_id=_format_gregor_id, @@ -513,20 +516,18 @@ def _process_participant_row(participant, phenotype_rows, missing_participant_id def _get_individual_data_types(projects): - sample_types = Sample.objects.filter(individual__family__project__in=projects).values_list('individual_id', 'sample_type') - individual_data_types = defaultdict(set) - for individual_db_id, sample_type in sample_types: - individual_data_types[individual_db_id].add(sample_type) - for individual_db_id in RnaSample.objects.filter(individual__family__project__in=projects).values_list('individual_id', flat=True): - individual_data_types[individual_db_id].add('RNA') - individuals = Individual.objects.filter(id__in=individual_data_types).prefetch_related( - 'family__project', 'mother', 'father') + rna_individuals = set(RnaSample.objects.filter(individual__family__project__in=projects).values_list('individual_id', flat=True)) + individuals = Individual.objects.filter(family__project__in=projects).annotate( + active_sample_types=ArrayAgg('active_datasets__sample_type', distinct=True, filter=Q(active_datasets__isnull=False)), + inactive_sample_types=ArrayAgg('inactive_datasets__sample_type', distinct=True, filter=Q(inactive_datasets__isnull=False)), + ).prefetch_related('family__project', 'mother', 'father') grouped_data_type_individuals = defaultdict(dict) for i in individuals: participant_id = _format_gregor_id(i.individual_id) - grouped_data_type_individuals[participant_id].update( - {data_type: i for data_type in individual_data_types[i.id]}) + data_types = {*i.active_sample_types, *i.inactive_sample_types} | ({'RNA'} if i.id in rna_individuals else set()) + if data_types: + grouped_data_type_individuals[participant_id].update({data_type: i for data_type in data_types}) return grouped_data_type_individuals @@ -995,7 +996,8 @@ def variant_metadata(request, project_guid): individuals = Individual.objects.filter( family__project__in=projects, family__savedvariant__varianttag__variant_tag_type__category=DISCOVERY_CATEGORY, ).distinct().annotate( - data_types=ArrayAgg('sample__sample_type', distinct=True, filter=Q(sample__isnull=False)) + active_data_types=ArrayAgg('active_datasets__sample_type', distinct=True), + inactive_data_types=ArrayAgg('inactive_datasets__sample_type', distinct=True), ) families_by_id = {} @@ -1021,8 +1023,8 @@ def _add_row(row, family_id, row_type): parse_anvil_metadata( projects, user=request.user, - individual_samples={i: None for i in individuals}, - individual_data_types={i.individual_id: i.data_types for i in individuals}, + individuals=individuals, + individual_data_types={i.individual_id: {*i.active_data_types, *i.inactive_data_types} for i in individuals}, add_row=_add_row, mme_value=ArrayAgg('matchmakersubmissiongenes__saved_variant__variant_id'), include_family_name_display=True, @@ -1069,6 +1071,6 @@ def _get_clickhouse_metadata(dataset_type, genome_version, keys, include_clinvar if include_clinvar: fields.append('clinvar') qs = qs.join_clinvar() - if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: + if dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS: fields.append('caid') return {(dataset_type, genome_version, v.pop('key')): v for v in qs.values(*fields)} diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 729b3ad93d..60d214a668 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -18,7 +18,7 @@ "id": "rec2B6OGmQpAkQW3s", "fields": { "SeqrCollaboratorSampleID": "VCGS_FAM203_621_D1", - "CollaboratorSampleID": "NA19675", + "CollaboratorSampleID": "NA19675_1", "Collaborator": ["recW24C2CJW5lT64K"], "dbgap_study_id": "dbgap_stady_id_1", "dbgap_subject_id": "dbgap_subject_id_1", @@ -746,7 +746,7 @@ def _check_anvil_export_response(self, response, mock_zip, no_analyst_project_ur 'entity:sample_id', '01-subject_id', '02-sample_id', '03-dbgap_sample_id', '04-sequencing_center', '05-sample_source', '06-tissue_affected_status',]) self.assertIn( - ['NA19675_1', 'NA19675_1', 'NA19675', 'SM-A4GQ4', 'Broad', '-', '-'], + ['NA19675_1', 'NA19675_1', 'NA19675_1', 'SM-A4GQ4', 'Broad', '-', '-'], sample_file, ) @@ -767,7 +767,7 @@ def _check_anvil_export_response(self, response, mock_zip, no_analyst_project_ur '1_248367227_HG00731', 'HG00731', 'HG00731', 'RP11', 'Known', 'paternal', 'Homozygous', 'GRCh37', '1', '248367227', 'TC', 'T', '-', '-', 'ENST00000371839', '-', '-', '-', '-'], discovery_file) self.assertIn([ - '21_3343353_NA19675_1', 'NA19675_1', 'NA19675', 'RP11', 'Candidate', 'de novo', + '21_3343353_NA19675_1', 'NA19675_1', 'NA19675_1', 'RP11', 'Candidate', 'de novo', 'Heterozygous', 'GRCh37', '21', '3343353', 'GAGA', 'G', 'c.375_377delTCT', 'p.Leu126del', 'ENST00000258436.5', '-', '-', '-', 'This individual is published in PMID34415322'], discovery_file) self.assertIn([ diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 6daa725fc7..81a3c0c713 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -4,7 +4,7 @@ from django.db.models import Q from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\ - Family, GeneNote, Project, Sample + Family, GeneNote, Project, Dataset from seqr.utils.xpos_utils import get_xpos from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json @@ -83,7 +83,7 @@ def create_manual_saved_variant_handler(request, family_guid): variant_json['saved_variant_json'] = {**variant_json} variant_json.update({ 'key': None, - 'dataset_type': Sample.DATASET_TYPE_SV_CALLS if variant_json.get('svName') else Sample.DATASET_TYPE_VARIANT_CALLS, + 'dataset_type': Dataset.DATASET_TYPE_SV_CALLS if variant_json.get('svName') else Dataset.DATASET_TYPE_VARIANT_CALLS, }) model_json = parse_saved_variant_json(variant_json, family.id) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 7382a100a2..6ed916b117 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -184,6 +184,18 @@ def test_saved_variant_data(self): tag = response_json['variantTagsByGuid']['VT1708633_2103343353_r0390_100'] self.assertSetEqual(set(tag.keys()), TAG_FIELDS) + self.assertDictEqual(tag, { + 'category': 'Collaboration', + 'color': '#668FE3', + 'createdBy': None, + 'lastModifiedDate': '2018-02-23T17:32:30.373Z', + 'metadata': None, + 'name': 'Review', + 'searchHash': '125897d6', + 'searchName': 'Incomplete Penetrance De-novo', + 'tagGuid': 'VT1708633_2103343353_r0390_100', + 'variantGuids': [VARIANT_GUID], + }) submissions = response_json['mmeSubmissionsByGuid'] self.assertSetEqual(set(submissions.keys()), {'MS000001_na19675'}) @@ -207,7 +219,7 @@ def test_saved_variant_data(self): 'outliers': { 'ENSG00000135953': [{ 'geneId': 'ENSG00000135953', 'zScore': 7.31, 'pValue': 0.00000000000948, 'pAdjust': 0.00000000781, - 'tissueType': 'M', 'isSignificant': True, + 'tissueType': 'M', 'sequencingType': 'T', 'isSignificant': True, }] }, 'spliceOutliers': {}, @@ -341,6 +353,7 @@ def test_saved_variant_data(self): 'category': 'CMG Discovery Tags', 'color': '#03441E', 'searchHash': None, + 'searchName': None, 'metadata': None, 'lastModifiedDate': '2018-05-29T16:32:51.449Z', 'createdBy': None, diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 6aaf14cb6a..6316d0167b 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -11,7 +11,7 @@ get_mme_metrics, get_hpo_terms_by_id from matchmaker.models import MatchmakerSubmission from reference_data.models import HumanPhenotypeOntology -from seqr.models import Project, Family, Individual, Sample, VariantTagType, SavedVariant, FamilyAnalysedBy +from seqr.models import Project, Family, Individual, Dataset, VariantTagType, SavedVariant, FamilyAnalysedBy from seqr.views.utils.airtable_utils import AirtableSession from seqr.views.utils.file_utils import load_uploaded_file from seqr.utils.communication_utils import safe_post_to_slack @@ -275,7 +275,7 @@ def _get_clickhouse_variants(families_by_id: dict[int, dict], family_variant_ids families_by_project[family['project__guid']].append(family['guid']) for project_guid, family_guids in families_by_project.items(): genotype_keys = get_clickhouse_genotypes( - project_guid, family_guids, genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variants_by_key.keys(), + project_guid, family_guids, genome_version, Dataset.DATASET_TYPE_VARIANT_CALLS, variants_by_key.keys(), additional_fields=['xpos'] ) for key, entry_data in genotype_keys.items(): diff --git a/seqr/views/apis/users_api_tests.py b/seqr/views/apis/users_api_tests.py index 7d7efe64a3..c45f86d892 100644 --- a/seqr/views/apis/users_api_tests.py +++ b/seqr/views/apis/users_api_tests.py @@ -388,7 +388,7 @@ def _test_set_password(self, set_password_url, password): # pylint: disable=argu self.assertEqual(response.reason_phrase, 'Password is required') response = self.client.post(set_password_url, content_type='application/json', data=json.dumps({ - 'userToken': quote_plus(password), 'password': 'password123', 'firstName': 'Test', 'isSuperuser': True})) + 'userToken': quote_plus(password), 'password': 'password123', 'firstName': 'Test', 'isSuperuser': True})) # nosec self.assertEqual(response.status_code, 200) user = User.objects.get(username='test_new_user') diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 3c7c9e2894..61f1748532 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -2,6 +2,7 @@ import jmespath from collections import defaultdict from copy import deepcopy +from datetime import timedelta from django.utils import timezone from django.contrib.postgres.aggregates import ArrayAgg from django.core.exceptions import MultipleObjectsReturned, PermissionDenied @@ -12,14 +13,15 @@ from math import ceil import re +from clickhouse_search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY +from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ + get_sorted_search_results, clickhouse_variant_lookup, InvalidSearchException from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_LOOKUP -from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Sample -from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, \ - variant_lookup, export_variants -from seqr.utils.search.utils import InvalidSearchException +from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Dataset from seqr.views.utils.export_utils import export_table from seqr.utils.gene_utils import get_genes_for_variant_display from seqr.utils.logging_utils import SeqrLogger +from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json from seqr.utils.xpos_utils import parse_variant_id from seqr.views.utils.json_utils import create_json_response, _to_snake_case from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ @@ -27,9 +29,9 @@ from seqr.views.utils.orm_to_json_utils import get_json_for_saved_variants_with_tags, get_json_for_saved_search,\ get_json_for_saved_searches, add_individual_hpo_details, FAMILY_ADDITIONAL_VALUES from seqr.views.utils.permissions_utils import check_project_permissions, get_project_guids_user_can_view, \ - login_and_policies_required, check_user_created_object_permissions, check_projects_view_permission + login_and_policies_required, check_user_created_object_permissions, check_projects_view_permission, user_is_analyst from seqr.views.utils.project_context_utils import get_projects_child_entities -from seqr.views.utils.variant_utils import get_variants_response +from seqr.views.utils.variant_utils import get_variants_response, variant_dataset_type from seqr.views.utils.vlm_utils import vlm_lookup logger = SeqrLogger(__name__) @@ -48,10 +50,11 @@ @login_and_policies_required def query_variants_handler(request, search_hash): - """Search variants. - """ page = int(request.GET.get('page') or 1) per_page = int(request.GET.get('per_page') or 100) + sort = request.GET.get('sort') or XPOS_SORT_KEY + if sort == PATHOGENICTY_SORT_KEY and user_is_analyst(request.user): + sort = PATHOGENICTY_HGMD_SORT_KEY search_context = json.loads(request.body or '{}') try: @@ -61,16 +64,42 @@ def query_variants_handler(request, search_hash): _check_results_permission(results_model, request.user) - variants, total_results = query_variants(results_model, sort=request.GET.get('sort'), page=page, num_results=per_page, user=request.user) + all_variants = _get_variants_with_cache( + _get_search_cache_key, _query_variants, results_model, request.user, sort=sort, + ) + variants = all_variants[(page-1)*per_page:page*per_page] response = _process_variants(variants or [], results_model.families.all(), request, genome_version=results_model.variant_search.search.get('no_access_project_genome_version')) response['search'] = _get_search_context(results_model) - response['search']['totalResults'] = total_results + response['search']['totalResults'] = len(all_variants) return create_json_response(response) +def _get_variants_with_cache(get_cache_key, get_variants, *args, **kwargs): + cache_key = get_cache_key(*args, **kwargs) + variants = safe_redis_get_json(cache_key) + if variants is None: + variants = get_variants(*args, **kwargs) + safe_redis_set_json(cache_key, variants, expire=timedelta(weeks=2)) + return variants + + +def _get_search_cache_key(results_model, user, sort=XPOS_SORT_KEY): + return 'search_results__{}__{}'.format(results_model.guid, sort) + + +def _query_variants(results_model, user, sort=XPOS_SORT_KEY): + families = results_model.families.all() + wildcard_cache_key = _get_search_cache_key(results_model, user, sort='*') + unsorted_variants = safe_redis_get_wildcard_json(wildcard_cache_key) + if unsorted_variants: + return get_sorted_search_results(unsorted_variants, sort, families) + + return get_clickhouse_variants(families, user, sort=sort, **results_model.variant_search.search) + + def _all_project_family_search_genome(search_context): return (search_context or {}).get('allGenomeProjectFamilies') @@ -107,15 +136,15 @@ def _get_or_create_results_model(search_hash, search_context, user): if search_context.get('unsolvedFamiliesOnly'): families = families.exclude(analysis_status__in=Family.SOLVED_ANALYSIS_STATUSES) if search_context.get('trioFamiliesOnly'): - families = families.annotate(search_sample_count=Count('individual__sample__id', filter=Q( - individual__sample__is_active=True, individual__sample__dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS, + families = families.annotate(search_sample_count=Count('individual__id', filter=Q( + individual__active_datasets__dataset_type=Dataset.DATASET_TYPE_VARIANT_CALLS, ))).filter( search_sample_count__gte=3, individual__mother__isnull=False, individual__father__isnull=False, ).distinct() search_dict = search_context.get('search', {}) if search_context.get('previousSearchHash') and (search_dict.get('exclude') or {}).get('previousSearch'): - search_dict['exclude']['previousSearchHash'] = search_context['previousSearchHash'] + search_dict.update(_get_exclude_keys(search_context['previousSearchHash'], user)) if include_no_access_projects: search_dict['no_access_project_genome_version'] = all_project_genome_version search_model = VariantSearch.objects.filter(search=search_dict).filter( @@ -134,17 +163,34 @@ def _get_or_create_results_model(search_hash, search_context, user): return results_model +def _get_exclude_keys(search_hash, user): + previous_results_model = VariantSearchResults.objects.get(search_hash=search_hash) + results = _get_variants_with_cache(_get_search_cache_key, _query_variants, previous_results_model, user) + exclude_keys = defaultdict(list) + exclude_key_pairs = defaultdict(list) + for variant in results: + if isinstance(variant, list): + dt1= variant_dataset_type(variant[0]) + dt2 = variant_dataset_type(variant[1]) + dataset_type = dt1 if dt1 == dt2 else ','.join(sorted([dt1, dt2])) + exclude_key_pairs[dataset_type].append(sorted([variant[0]['key'], variant[1]['key']])) + else: + dataset_type = variant_dataset_type(variant) + exclude_keys[dataset_type].append(variant['key']) + return {'exclude_keys': dict(exclude_keys), 'exclude_key_pairs': dict(exclude_key_pairs)} + + @login_and_policies_required def query_single_variant_handler(request, variant_id): - """Search variants. - """ families = Family.objects.filter(guid=request.GET.get('familyGuid')) family = families.first() check_project_permissions(family.project, request.user) - variant = get_single_variant(families, variant_id, request.user) + variants = get_clickhouse_variants(families, request.user, raw_variant_items=variant_id, variant_ids=[variant_id]) + if not variants: + raise InvalidSearchException('Variant {} not found'.format(variant_id)) - response = _process_variants([variant], families, request, add_all_context=True, add_locus_list_detail=True) + response = _process_variants(variants, families, request, add_all_context=True, add_locus_list_detail=True) return create_json_response(response) @@ -153,6 +199,7 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo if not variants: return {'searchedVariantIds': [], 'variantsById': {}} + variants = format_clickhouse_results(variants) flat_variants = _flatten_variants(variants) variants_by_id = {v['variantId']: v for v in flat_variants} saved_variants = _get_saved_variant_models(flat_variants) @@ -233,6 +280,7 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo {'header': 'ab'}, ] +MAX_EXPORT_VARIANTS = 1000 MAX_FAMILIES_PER_ROW = 1000 @@ -241,7 +289,18 @@ def get_variant_gene_breakdown(request, search_hash): results_model = VariantSearchResults.objects.get(search_hash=search_hash) projects = _check_results_permission(results_model, request.user) - gene_counts = get_variant_query_gene_counts(results_model, user=request.user) + results = _get_variants_with_cache(_get_search_cache_key, _query_variants, results_model, request.user) + flat_variants = [ + v for variants in results for v in (variants if isinstance(variants, list) else [variants]) + ] + gene_counts = defaultdict(lambda: {'total': 0, 'families': defaultdict(int)}) + for var in flat_variants: + gene_ids = var['transcripts'].keys() if 'transcripts' in var else {t['geneId'] for t in var['sortedTranscriptConsequences']} + for gene_id in gene_ids: + gene_counts[gene_id]['total'] += 1 + for family_guid in var['familyGuids']: + gene_counts[gene_id]['families'][family_guid] += 1 + return create_json_response({ 'searchGeneBreakdown': {search_hash: gene_counts}, 'genesById': get_genes_for_variant_display(list(gene_counts.keys()), projects.first().genome_version), @@ -258,7 +317,12 @@ def export_variants_handler(request, search_hash): families = results_model.families.all() family_ids_by_guid = {family.guid: family.family_id for family in families} - variants = export_variants(results_model, request.user) + variants = _get_variants_with_cache(_get_search_cache_key, _query_variants, results_model, request.user) + total_variants = len(variants) + if total_variants > MAX_EXPORT_VARIANTS: + raise InvalidSearchException(f'Unable to export more than {MAX_EXPORT_VARIANTS} variants ({total_variants} requested)') + + variants = format_clickhouse_export_results(variants) saved_variants = _get_saved_variant_models(variants) json_saved_variants = get_json_for_saved_variants_with_tags(saved_variants) @@ -336,8 +400,6 @@ def _get_field_value(value, config): @login_and_policies_required def search_context_handler(request): - """Search variants. - """ response = _get_saved_searches(request.user) context = json.loads(request.body) @@ -371,16 +433,16 @@ def search_context_handler(request): projectGuid=Value(project_guid) if project_guid else F('project__guid'), familyGuid=F('guid'), analysisStatus=F('analysis_status'), + sampleTypes=ArrayAgg( + JSONObject(sampleType='individual__active_datasets__sample_type', datasetType='individual__active_datasets__dataset_type', isActive=Value(True)), + distinct=True, filter=Q(individual__active_datasets__dataset_type__isnull=False), + ), **FAMILY_ADDITIONAL_VALUES, )} - family_sample_types = Sample.objects.filter(individual__family__project__in=projects, is_active=True).values('individual__family__guid').annotate( - samples=ArrayAgg(JSONObject(sampleType='sample_type', datasetType='dataset_type', isActive=Value(True)), distinct=True)) project_dataset_types = defaultdict(set) - for agg in family_sample_types: - family = response['familiesByGuid'][agg['individual__family__guid']] - family['sampleTypes'] = agg['samples'] - project_dataset_types[family['projectGuid']].update([s['datasetType'] for s in agg['samples']]) + for family in response['familiesByGuid'].values(): + project_dataset_types[family['projectGuid']].update([s['datasetType'] for s in family['sampleTypes']]) for project_guid, dataset_types in project_dataset_types.items(): response['projectsByGuid'][project_guid]['datasetTypes'] = list(dataset_types) @@ -528,7 +590,10 @@ def variant_lookup_handler(request): variant_id = request.GET.get('variantId') genome_version = request.GET.get('genomeVersion') or GENOME_VERSION_GRCh38 bool_kwargs = {_to_snake_case(field): bool(request.GET.get(field)) for field in ['affectedOnly', 'homOnly']} - variants = variant_lookup(request.user, variant_id, genome_version, sample_type=request.GET.get('sampleType'), **bool_kwargs) + variants = _get_variants_with_cache( + _get_lookup_cache_key, clickhouse_variant_lookup, + request.user, variant_id, sample_type=request.GET.get('sampleType'), genome_version=genome_version, **bool_kwargs, + ) family_guids = set() for variant in variants: @@ -556,6 +621,15 @@ def variant_lookup_handler(request): return create_json_response(response) +def _get_lookup_cache_key(user, variant_id, sample_type, genome_version, affected_only, hom_only): + cache_fields = ['variant_lookup_results', variant_id, genome_version] + if affected_only: + cache_fields.append('affected') + if hom_only: + cache_fields.append('hom') + return '__'.join(cache_fields) + + def _update_lookup_variant(variant, response, individual_guid_map, user): no_access_families = set(variant['familyGenotypes']) - set(variant['familyGuids']) individual_summary_map = { diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 631db9c59a..3757d58ddc 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -23,6 +23,7 @@ EXPECTED_SEARCH_CONTEXT_RESPONSE = { 'savedSearchesByGuid': { 'VS0079516_': mock.ANY, 'VS0079525_': mock.ANY, 'VS0079517_': mock.ANY, 'VS0145435_': mock.ANY, + 'VS0001234_incomplete_penetranc': mock.ANY, }, 'projectsByGuid': {PROJECT_GUID: mock.ANY}, 'familiesByGuid': mock.ANY, @@ -211,7 +212,7 @@ def test_search_context(self): expected_response['projectsByGuid']['R0003_test'] = mock.ANY self.assertSetEqual(set(response_json), set(expected_response)) self.assertDictEqual(response_json, expected_response) - self.assertEqual(len(response_json['savedSearchesByGuid']), 4) + self.assertEqual(len(response_json['savedSearchesByGuid']), 5) self.assertSetEqual(set(response_json['projectsByGuid'][PROJECT_GUID].keys()), PROJECT_CONTEXT_FIELDS) self.assertSetEqual(set(response_json['projectsByGuid'][PROJECT_GUID]['datasetTypes']), {'SNV_INDEL', 'SV', 'MITO'}) self.assertSetEqual(set(response_json['projectsByGuid']['R0003_test']['datasetTypes']), {'SNV_INDEL'}) @@ -344,7 +345,7 @@ def test_saved_search(self): response = self.client.get(get_saved_search_url) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.json()['savedSearchesByGuid']), 4) + self.assertEqual(len(response.json()['savedSearchesByGuid']), 5) create_saved_search_url = reverse(create_saved_search_handler) @@ -379,7 +380,7 @@ def test_saved_search(self): response = self.client.get(get_saved_search_url) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.json()['savedSearchesByGuid']), 5) + self.assertEqual(len(response.json()['savedSearchesByGuid']), 6) # Test cannot save different searches with the same name body['filters'] = {'test': 'filter'} @@ -409,7 +410,7 @@ def test_saved_search(self): response = self.client.get(get_saved_search_url) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.json()['savedSearchesByGuid']), 4) + self.assertEqual(len(response.json()['savedSearchesByGuid']), 5) global_saved_search_guid = next(iter(response.json()['savedSearchesByGuid'])) diff --git a/seqr/views/utils/airtable_utils.py b/seqr/views/utils/airtable_utils.py index ca806afe0f..3d57e5d5f3 100644 --- a/seqr/views/utils/airtable_utils.py +++ b/seqr/views/utils/airtable_utils.py @@ -207,7 +207,7 @@ def get_samples_for_matched_pdos(self, pdo_statuses, pdo_fields=None, sample_fie project_guids = [match and match.group(1) for match in project_matches] pdos = [{ 'project_guid': project_guids[i] if len(project_guids) > 1 else project_guids[0], - **{field: sample[field][i] for field in pdo_fields} + **{field: sample[field][i] if len(sample[field]) == len(sample['PDOStatus']) else None for field in pdo_fields} } for i, status in enumerate(sample['PDOStatus']) if status in pdo_statuses] if project_guid: pdos = [pdo for pdo in pdos if pdo['project_guid'] == project_guid] diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 5d9d860ab0..53ea3b4bcd 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -1,8 +1,8 @@ from collections import defaultdict from datetime import datetime -from django.db.models import F, Q, Value, Case, When, CharField, Aggregate -from django.db.models.functions import Replace +from django.db.models import F, Q, Value, Case, When, CharField, Aggregate, Min +from django.db.models.functions import Replace, Coalesce from django.contrib.auth.models import User from django.contrib.postgres.aggregates import ArrayAgg import requests @@ -10,7 +10,7 @@ from matchmaker.models import MatchmakerSubmission from reference_data.models import HumanPhenotypeOntology, Omim, GENOME_VERSION_LOOKUP -from seqr.models import Project, Family, Individual, Sample, SavedVariant, VariantTagType +from seqr.models import Project, Family, Individual, Dataset, SavedVariant, VariantTagType from seqr.views.utils.airtable_utils import AirtableSession from seqr.utils.gene_utils import get_genes from seqr.utils.middleware import ErrorsWarningsException @@ -96,8 +96,8 @@ } METHOD_MAP = { - Sample.SAMPLE_TYPE_WES: 'SR-ES', - Sample.SAMPLE_TYPE_WGS: 'SR-GS', + Dataset.SAMPLE_TYPE_WES: 'SR-ES', + Dataset.SAMPLE_TYPE_WGS: 'SR-GS', } FAMILY_INDIVIDUAL_FIELDS = ['family_id', 'internal_project_id', 'phenotype_description', 'pmid_id', 'solve_status'] @@ -163,31 +163,31 @@ def _get_variant_json(saved_variants, fields, annotations): def parse_anvil_metadata( projects: Iterable[Project], user: User, add_row: Callable[[dict, str, str], None], max_loaded_date: str = None, family_fields: dict = None, format_id: Callable[[str], str] = lambda s: s, - get_additional_sample_fields: Callable[[Sample, dict], dict] = None, + get_additional_sample_fields: Callable[[Individual, dict], dict] = None, get_additional_individual_fields: Callable[[Individual, dict], dict] = None, - individual_samples: dict[Individual, Sample] = None, individual_data_types: dict[str, Iterable[str]] = None, + individuals: list[Individual] = None, individual_data_types: dict[str, Iterable[str]] = None, airtable_fields: Iterable[str] = None, mme_value: Aggregate = None, get_variant_json: Callable[[Iterable[SavedVariant], list[str], dict], dict] = _get_variant_json, post_process_variant: Callable[[dict, list[dict]], dict] = None, include_no_individual_families: bool = False, omit_airtable: bool = False, include_family_name_display: bool = False, include_family_sample_metadata: bool = False, include_discovery_sample_id: bool = False, include_mondo: bool = False, omit_parent_mnvs: bool = False, proband_only_variants: bool = False): - individual_samples = individual_samples or (_get_loaded_before_date_project_individual_samples(projects, max_loaded_date) \ - if max_loaded_date else _get_all_project_individual_samples(projects)) + if not individuals: + individuals = _get_sample_annotated_individuals(projects, max_loaded_date) family_data_by_id = _get_family_metadata( - {'project__in': projects} if include_no_individual_families else {'individual__in': individual_samples}, + {'project__in': projects} if include_no_individual_families else {'individual__in': individuals}, family_fields, include_family_name_display, include_family_sample_metadata, include_mondo, format_id ) individuals_by_family_id = defaultdict(list) individual_ids_map = {} sample_ids = set() - for individual, sample in individual_samples.items(): + for individual in individuals: individuals_by_family_id[individual.family_id].append(individual) individual_ids_map[individual.id] = (individual.individual_id, individual.guid) - if sample: - sample_ids.add(sample.sample_id) + if getattr(individual, 'sample_type', None): + sample_ids.add(individual.individual_id) saved_variants_by_family = _get_parsed_saved_discovery_variants_by_family( list(family_data_by_id.keys()), bool(mme_value), get_variant_json, @@ -199,7 +199,7 @@ def parse_anvil_metadata( list(sample_ids) or [i[0] for i in individual_ids_map.values()], user, airtable_fields) matchmaker_individuals = {m['individual_id']: m['value'] for m in MatchmakerSubmission.objects.filter( - individual__in=individual_samples).values('individual_id', value=mme_value)} if mme_value else {} + individual__in=individuals).values('individual_id', value=mme_value)} if mme_value else {} for family_id, family_subject_row in family_data_by_id.items(): saved_variants = saved_variants_by_family[family_id] @@ -223,15 +223,14 @@ def parse_anvil_metadata( add_row(family_row, family_id, FAMILY_ROW_TYPE) for individual in family_individuals: - sample = individual_samples[individual] - airtable_metadata = None has_dbgap_submission = None + sample_type = getattr(individual, 'sample_type', None) if sample_airtable_metadata is not None: - if sample: - airtable_metadata = sample_airtable_metadata.get(sample.sample_id, {}) + if sample_type: + airtable_metadata = sample_airtable_metadata.get(individual.individual_id, {}) dbgap_submission = airtable_metadata.get('dbgap_submission') or set() - has_dbgap_submission = sample.sample_type in dbgap_submission + has_dbgap_submission = sample_type in dbgap_submission elif not sample_ids: airtable_metadata = sample_airtable_metadata.get(individual.individual_id, {}) @@ -249,8 +248,8 @@ def parse_anvil_metadata( add_row(subject_row, family_id, SUBJECT_ROW_TYPE) participant_id = subject_row['participant_id'] - if sample: - sample_row = _get_sample_row(sample, participant_id, has_dbgap_submission, airtable_metadata, include_family_sample_metadata, get_additional_sample_fields) + if sample_type: + sample_row = _get_sample_row(individual, participant_id, has_dbgap_submission, airtable_metadata, include_family_sample_metadata, get_additional_sample_fields) add_row(sample_row, family_id, SAMPLE_ROW_TYPE) if proband_only_variants and individual.proband_relationship != Individual.SELF_RELATIONSHIP: @@ -260,7 +259,7 @@ def parse_anvil_metadata( format_id=format_id, omit_parent_mnvs=omit_parent_mnvs, individual_data_types=(individual_data_types or {}).get(participant_id), family_individuals=family_individuals if proband_only_variants else None, - sample=sample if include_discovery_sample_id else None, + sample_id=individual.individual_id if include_discovery_sample_id else None, post_process_variant=post_process_variant, ) add_row(discovery_row, family_id, DISCOVERY_ROW_TYPE) @@ -270,22 +269,24 @@ def _get_nested_variant_name(v): return v['sv_name'] or f"{v['chrom']}-{v['pos']}-{v['ref']}-{v['alt']}" -def _get_loaded_before_date_project_individual_samples(projects, max_loaded_date): - max_loaded_date = datetime.strptime(max_loaded_date, '%Y-%m-%d') - loaded_samples = _get_sorted_search_samples(projects).filter( - loaded_date__lte=max_loaded_date).select_related('individual') - # Only return the oldest sample for each individual - return {sample.individual: sample for sample in loaded_samples} - - -def _get_all_project_individual_samples(projects): - samples_by_individual_id = {s.individual_id: s for s in _get_sorted_search_samples(projects)} +def _get_sample_annotated_individuals(projects, max_loaded_date): individuals = Individual.objects.filter(family__project__in=projects) - return {i: samples_by_individual_id.get(i.id) for i in individuals} - - -def _get_sorted_search_samples(projects): - return Sample.objects.filter(individual__family__project__in=projects).order_by('-loaded_date') + active_filter = inactive_filter = None + if max_loaded_date: + max_loaded_date = datetime.strptime(max_loaded_date, '%Y-%m-%d') + active_filter = Q(active_datasets__loaded_date__lte=max_loaded_date) + inactive_filter = Q(inactive_datasets__loaded_date__lte=max_loaded_date) + individuals = individuals.filter(active_filter | inactive_filter) + return individuals.annotate( + sample_type=Coalesce( + Min('active_datasets__sample_type', filter=active_filter), + Min('inactive_datasets__sample_type', filter=inactive_filter), + ), + loaded_date=Coalesce( + Min('inactive_datasets__loaded_date', filter=inactive_filter), + Min('active_datasets__loaded_date', filter=active_filter), + ), + ) HET = 'Heterozygous' @@ -458,27 +459,27 @@ def anvil_export_airtable_fields(airtable_metadata, has_dbgap_submission): } -def _get_sample_row(sample, participant_id, has_dbgap_submission, airtable_metadata, include_family_sample_metadata, get_additional_sample_fields=None): +def _get_sample_row(individual, participant_id, has_dbgap_submission, airtable_metadata, include_family_sample_metadata, get_additional_sample_fields=None): sample_row = { 'participant_id': participant_id, - 'sample_id': sample.sample_id, + 'sample_id': individual.individual_id, } if has_dbgap_submission: sample_row['dbgap_sample_id'] = airtable_metadata.get('dbgap_sample_id', '') if include_family_sample_metadata: sample_row.update({ - 'data_type': sample.sample_type, - 'date_data_generation': sample.loaded_date.strftime('%Y-%m-%d'), + 'data_type': individual.sample_type, + 'date_data_generation': individual.loaded_date.strftime('%Y-%m-%d'), }) if get_additional_sample_fields: - sample_row.update(get_additional_sample_fields(sample, airtable_metadata)) + sample_row.update(get_additional_sample_fields(individual, airtable_metadata)) return sample_row def _get_genetic_findings_rows(rows: list[dict], individual: Individual, family_row: dict, participant_id: str, individual_data_types: Iterable[str], family_individuals: dict[str, str], post_process_variant: Callable[[dict, list[dict]], dict], - format_id: Callable[[str], str], omit_parent_mnvs: bool, sample: Sample) -> list[dict]: + format_id: Callable[[str], str], omit_parent_mnvs: bool, sample_id: str) -> list[dict]: parsed_rows = [] variants_by_gene = defaultdict(list) for row in (rows or []): @@ -510,10 +511,10 @@ def _get_genetic_findings_rows(rows: list[dict], individual: Individual, family_ ]) if individual_data_types is not None: parsed_row['method_of_discovery'] = '|'.join([ - METHOD_MAP.get(data_type) for data_type in individual_data_types if data_type in Sample.SAMPLE_TYPE_LOOKUP + METHOD_MAP.get(data_type) for data_type in individual_data_types if data_type in Dataset.SAMPLE_TYPE_LOOKUP ]) - if sample is not None: - parsed_row['sample_id'] = sample.sample_id + if sample_id is not None: + parsed_row['sample_id'] = sample_id parsed_rows.append(parsed_row) variants_by_gene[row[GENE_COLUMN]].append({**parsed_row, 'individual_genotype': individual_genotype}) diff --git a/seqr/views/utils/dataset_utils.py b/seqr/views/utils/dataset_utils.py index f0a5571cf2..90071f8bae 100644 --- a/seqr/views/utils/dataset_utils.py +++ b/seqr/views/utils/dataset_utils.py @@ -11,7 +11,7 @@ from seqr.utils.file_utils import file_iter, is_google_bucket_file_path, run_gsutil_with_wait from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException -from seqr.utils.search.add_data_utils import basic_notify_search_data_loaded +from seqr.utils.add_data_utils import basic_notify_search_data_loaded from seqr.utils.xpos_utils import format_chrom from seqr.views.utils.file_utils import parse_file, get_temp_file_path, persist_temp_file from seqr.views.utils.json_utils import _to_snake_case, _to_camel_case @@ -23,8 +23,8 @@ def _create_rna_samples(sample_data, sample_guid_ids_to_load, user, **kwargs): new_samples = [ - RnaSample(individual_id=individual_id, tissue_type=tissue_type, **kwargs) - for individual_id, tissue_type in sample_data.items() + RnaSample(individual_id=individual_id, tissue_type=tissue_type, sequencing_type=sequencing_type, **kwargs) + for individual_id, (tissue_type, sequencing_type) in sample_data.items() ] new_sample_models = RnaSample.bulk_create(user, new_samples) new_sample_ids = [s.id for s in new_sample_models] @@ -34,7 +34,7 @@ def _create_rna_samples(sample_data, sample_guid_ids_to_load, user, **kwargs): def _get_rna_sample_data_by_key(values=None, **kwargs): - key_fields = ['individual_id', 'tissue_type'] + key_fields = ['individual_id', 'tissue_type', 'sequencing_type'] return { tuple(s.pop(k) for k in key_fields): s for s in RnaSample.objects.filter(**kwargs).values('guid', *key_fields, **(values or {})) @@ -191,7 +191,7 @@ def _load_rna_seq_file( ) inactivate_samples = { sample['guid']: key[0] for key, sample in potential_inactivate_samples_by_key.items() - if key in set(samples_to_create.items()) and sample['guid'] not in loaded_samples + if key in {tuple([k, *v]) for k, v in samples_to_create.items()} and sample['guid'] not in loaded_samples } errors, warnings = _process_rna_errors( @@ -223,7 +223,8 @@ def _parse_rna_row(sample_id, row_dict, potential_samples, loaded_samples, gene_ return tissue_type = individual['tissue'] - potential_sample = potential_samples.get((individual['id'], tissue_type)) + sequencing_type = individual['sequencing_type'] + potential_sample = potential_samples.get((individual['id'], tissue_type, sequencing_type)) if (potential_sample or {}).get('active'): loaded_samples.add(potential_sample['guid']) return @@ -231,7 +232,7 @@ def _parse_rna_row(sample_id, row_dict, potential_samples, loaded_samples, gene_ if potential_sample: sample_guid_ids_to_load[potential_sample['guid']] = sample_id else: - samples_to_create[individual['id']] = tissue_type + samples_to_create[individual['id']] = (tissue_type, sequencing_type) if has_errors: # If there are definite errors, do not process/save data, just continue to check for additional errors @@ -319,7 +320,7 @@ def load_rna_seq(request_json, user, **kwargs): try: sample_guids, file_name_prefix, info, warnings = _load_rna_seq( - data_type, file_path, user, **kwargs, + data_type, file_path, user, **kwargs, sequencing_type=request_json.get('sequencingType'), tissue=request_json.get('tissue'), ignore_extra_samples=request_json.get('ignoreExtraSamples'), skip_new_sample_validation=request_json.get('skipNewSampleValidation'), ) @@ -336,7 +337,7 @@ def load_rna_seq(request_json, user, **kwargs): }, 200 -def _load_rna_seq(data_type, file_path, user, sample_metadata_mapping=None, project_guid=None, tissue=None, **kwargs): +def _load_rna_seq(data_type, file_path, user, sample_metadata_mapping=None, project_guid=None, sequencing_type=None, tissue=None, **kwargs): config = RNA_DATA_TYPE_CONFIGS[data_type] model_cls = config['model_class'] file_name = file_path.split('/')[-1] @@ -349,7 +350,7 @@ def _load_rna_seq(data_type, file_path, user, sample_metadata_mapping=None, proj individuals = Individual.objects.filter(family__project__in=projects) individual_data_by_id = _get_individual_metadata_mapping(sample_metadata_mapping, individuals) if sample_metadata_mapping else { - i['individual_id']: {**i, 'tissue': tissue} for i in individuals.values('id', 'individual_id') + i['individual_id']: {**i, 'tissue': tissue, 'sequencing_type': sequencing_type} for i in individuals.values('id', 'individual_id') } potential_samples = _get_rna_sample_data_by_key( individual_id__in={i['id'] for i in individual_data_by_id.values()}, @@ -415,7 +416,7 @@ def _get_individual_metadata_mapping(sample_metadata_mapping, individuals): individual_id = indiv['individual_id'] sample_metadata = sample_metadata_mapping[individual_id] if indiv.pop('family__project__guid') == sample_metadata['project_guid']: - individual_data = {**indiv, 'tissue': sample_metadata['tissue']} + individual_data = {**indiv, 'tissue': sample_metadata['tissue'], 'sequencing_type': sample_metadata['sequencing_type']} individual_data_by_id[individual_id] = individual_data # Support when data is provided using either the raw sample ID or the already mapped seqr ID if individual_id != sample_metadata['sample_id']: diff --git a/seqr/views/utils/export_utils.py b/seqr/views/utils/export_utils.py index df0d51cfaf..92992a6209 100644 --- a/seqr/views/utils/export_utils.py +++ b/seqr/views/utils/export_utils.py @@ -24,8 +24,10 @@ def export_table(filename_prefix, header, rows, file_format='tsv', titlecase_hea header (list): List of column names rows (list): List of rows, where each row is a list of column values file_format (string): "tsv", "xls", or "json" + Returns: Django HttpResponse object with the table data as an attachment. + """ for i, row in enumerate(rows): diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index b9e46d1a21..6007ff8d6d 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -1,13 +1,10 @@ -""" -APIs for retrieving, updating, creating, and deleting Individual records -""" from collections import defaultdict -from clickhouse_search.models.postgres_dicts import SexDict +from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict from matchmaker.models import MatchmakerSubmission, MatchmakerResult -from seqr.models import Sample, IgvSample, RnaSample, Individual, Family, FamilyNote +from seqr.models import Dataset, IgvSample, RnaSample, Individual, Family, FamilyNote from seqr.utils.middleware import ErrorsWarningsException -from seqr.utils.search.add_data_utils import trigger_rebuild_gt_stats +from seqr.utils.add_data_utils import trigger_rebuild_gt_stats from seqr.views.utils.json_to_orm_utils import update_individual_from_json, update_individual_parents, create_model_from_json, \ update_family_from_json from seqr.views.utils.orm_to_json_utils import _get_json_for_individuals, _get_json_for_families, get_json_for_family_notes @@ -24,19 +21,6 @@ def _get_record_individual_id(record): def add_or_update_individuals_and_families(project, individual_records, user, get_update_json=True, get_updated_individual_db_ids=False, get_created_counts=False, allow_features_update=False, skip_gt_stats_rebuild=False): - """ - Add or update individual and family records in the given project. - - Args: - project (object): Django ORM model for the project to add families to - individual_records (list): A list of JSON records representing individuals. See - the return value of pedigree_info_utils#convert_fam_file_rows_to_json(..) - user (object): current user model - - Return: - 3-tuple: updated Individual models, updated Family models, and updated FamilyNote models - - """ updated_family_ids = set() updated_individuals = set() updated_affected = set() @@ -86,10 +70,12 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge updated_family_models = Family.objects.filter(id__in=updated_family_ids) _remove_pedigree_images(updated_family_models, user) - if updated_affected and not skip_gt_stats_rebuild: - trigger_rebuild_gt_stats(project, user) + if updated_affected: + AffectedDict.reload(user) + if not skip_gt_stats_rebuild: + trigger_rebuild_gt_stats(project, user) if updated_sex: - SexDict.reload() + SexDict.reload(user) pedigree_json = None if get_update_json: @@ -180,20 +166,19 @@ def _update_from_record(record, user, families_by_id, individual_lookup, updated def delete_individuals(project, individual_guids, user): - """Delete one or more individuals - - Args: - project (object): Django ORM model for project - individual_guids (list): GUIDs of individuals to delete - - Returns: - list: Family objects for families with deleted individuals - """ errors, individuals_to_delete = check_project_individuals_deletable(project, individual_guids=individual_guids) if errors: raise ErrorsWarningsException(errors) - Sample.bulk_delete(user, individual__in=individuals_to_delete) + datasets = Dataset.objects.filter(inactive_individuals__in=individuals_to_delete) + if individual_guids is None: + Dataset.bulk_delete(user, queryset=datasets) + else: + for dataset in datasets: + dataset.inactive_individuals.remove(*individuals_to_delete) + if not dataset.inactive_individuals.exists(): + dataset.delete_model(user, user_can_delete=True) + IgvSample.bulk_delete(user, individual__in=individuals_to_delete) RnaSample.bulk_delete(user, individual__in=individuals_to_delete) MatchmakerResult.bulk_delete(user, submission__individual__in=individuals_to_delete, submission__deleted_date__isnull=False) @@ -231,7 +216,7 @@ def _validate_no_sumissions_no_search_samples(individuals_to_delete): individuals_to_delete, 'MME submission', dict(matchmakersubmission__isnull=False, matchmakersubmission__deleted_date__isnull=True) ) + _validate_delete_individuals( - individuals_to_delete, 'search sample', dict(sample__is_active=True) + individuals_to_delete, 'search sample', dict(active_datasets__isnull=False) ) @@ -244,7 +229,7 @@ def _get_updated_pedigree_json(updated_individuals, updated_families, updated_no individual['individualGuid']: individual for individual in _get_json_for_individuals(Individual.objects.filter(id__in=[ i.id for i in updated_individuals - ]), user, add_sample_guids_field=True) + ]), user) } families_by_guid = { family['familyGuid']: family for family in diff --git a/seqr/views/utils/json_utils.py b/seqr/views/utils/json_utils.py index 60cd6b77d0..78bad25f44 100644 --- a/seqr/views/utils/json_utils.py +++ b/seqr/views/utils/json_utils.py @@ -17,15 +17,6 @@ def default(self, o): def create_json_response(obj, **kwargs): - """Encodes the give object into json and create a django JsonResponse object with it. - - Args: - obj (object): json response object - **kwargs: any addition args to pass to the JsonResponse constructor - Returns: - JsonResponse - """ - dumps_params = { 'sort_keys': True, 'indent': 4, @@ -40,7 +31,6 @@ def create_json_response(obj, **kwargs): def _to_camel_case(snake_case_str): - """Convert snake_case string to CamelCase""" if not CAMEL_CASE_MAP.get(snake_case_str): converted = snake_case_str.replace('_', ' ').title().replace(' ', '') CAMEL_CASE_MAP[snake_case_str] = converted[0].lower() + converted[1:] @@ -48,13 +38,10 @@ def _to_camel_case(snake_case_str): def _to_title_case(snake_case_str): - """Convert snake_case string to Title Case""" - components = snake_case_str.split('_') return " ".join(x.title() for x in components) def _to_snake_case(camel_case_str): - """Convert CamelCase string to snake_case (from https://gist.github.com/jaytaylor/3660565)""" - + # from https://gist.github.com/jaytaylor/3660565 return re.sub('([A-Z])', '_\\1', camel_case_str).lower().lstrip('_') diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 3f2c153c22..34d3c2616a 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -1,10 +1,6 @@ -""" -Utility functions for converting Django ORM object to JSON -""" - from collections import defaultdict from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models import prefetch_related_objects, Count, Value, F, Q, CharField, Case, When +from django.db.models import prefetch_related_objects, Count, Value, F, Q, CharField, Case, When, OuterRef, Subquery from django.db.models.functions import Concat, Coalesce, NullIf, Lower, Trim, JSONObject from django.contrib.auth.models import User from guardian.shortcuts import get_users_with_perms, get_groups_with_perms @@ -13,7 +9,7 @@ from panelapp.models import PaLocusList from reference_data.models import HumanPhenotypeOntology from seqr.models import GeneNote, VariantNote, VariantTag, VariantFunctionalData, SavedVariant, Family, CAN_VIEW, CAN_EDIT, \ - get_audit_field_names, RnaSeqOutlier, RnaSeqSpliceOutlier + get_audit_field_names, RnaSeqOutlier, RnaSeqSpliceOutlier, VariantSearchResults from seqr.utils.xpos_utils import get_chrom_pos from seqr.views.utils.json_utils import _to_camel_case from seqr.views.utils.permissions_utils import has_project_permissions, \ @@ -41,18 +37,6 @@ def _get_model_json_fields(model_class, user, is_analyst, additional_model_field def _get_json_for_models(models, nested_fields=None, user=None, is_analyst=None, process_result=None, guid_key=None, additional_model_fields=None): - """Returns an array JSON representations of the given models. - - Args: - models (array): Array of django models - user (object): Django User object for determining whether to include restricted/internal-only fields - nested_fields (array): Optional array of fields to get from the model that are nested on related objects - process_result (lambda): Optional function to post-process a given model json - guid_key (string): Optional key to use for the model's guid - Returns: - array: json objects - """ - if not models: return [] @@ -93,14 +77,6 @@ def _get_json_for_models(models, nested_fields=None, user=None, is_analyst=None, def _get_json_for_model(model, get_json_for_models=_get_json_for_models, **kwargs): - """Helper function to return a JSON representations of the given model. - - Args: - model (object): Django models - get_json_for_models (lambda): Function used to determine the json for an array of the given model - Returns: - object: json object - """ return get_json_for_models([model], **kwargs)[0] @@ -171,14 +147,6 @@ def get_json_for_current_user(user): def get_json_for_projects(projects, user=None, is_analyst=None, add_project_category_guids_field=True, add_permissions=False): - """Returns JSON representation of the given Projects. - - Args: - projects (object array): Django models for the projects - user (object): Django User object for determining whether to include restricted/internal-only fields - Returns: - dict: json object - """ def _process_result(result, project): result.update({ 'isMmeEnabled': result['isMmeEnabled'] and not result['isDemo'], @@ -198,14 +166,6 @@ def _process_result(result, project): def _get_json_for_project(project, user, **kwargs): - """Returns JSON representation of the given Project. - - Args: - project (object): Django model for the project - user (object): Django User object for determining whether to include restricted/internal-only fields - Returns: - dict: json object - """ return _get_json_for_model(project, get_json_for_models=get_json_for_projects, user=user, add_permissions=True, **kwargs) @@ -221,7 +181,7 @@ def _get_case_review_fields(model_cls, has_case_review_perm): createdBy=_user_expr('familyanalysedby__created_by'), dataType='familyanalysedby__data_type', lastModifiedDate='familyanalysedby__last_modified_date', - ), filter=Q(familyanalysedby__isnull=False)), + ), filter=Q(familyanalysedby__isnull=False), distinct=True), 'assignedAnalyst': Case( When(assigned_analyst__isnull=False, then=JSONObject( fullName=_full_name_expr('assigned_analyst'), email=F('assigned_analyst__email'), @@ -265,7 +225,7 @@ def get_json_for_family_note(note): INDIVIDUAL_DISPLAY_NAME_EXPR = Coalesce(NullIf('display_name', Value('')), 'individual_id', output_field=CharField()) -def _get_json_for_individuals(individuals, user=None, project_guid=None, add_sample_guids_field=False, +def _get_json_for_individuals(individuals, user=None, project_guid=None, add_hpo_details=False, is_analyst=None, has_case_review_perm=False): additional_model_fields = _get_case_review_fields(individuals.model, has_case_review_perm) nested_fields = [ @@ -284,11 +244,6 @@ def _get_json_for_individuals(individuals, user=None, project_guid=None, add_sam 'paternalId': F('father__individual_id'), 'displayName': INDIVIDUAL_DISPLAY_NAME_EXPR, } - if add_sample_guids_field: - additional_values.update({ - f'{field}Guids': ArrayAgg(f'{field.lower()}__guid', filter=Q(**{f'{field.lower()}__isnull': False})) - for field in ['sample', 'igvSample'] - }) parsed_individuals = get_json_for_queryset( individuals, user=user, is_analyst=is_analyst, additional_values=additional_values, @@ -328,39 +283,22 @@ def _get_sample_json_kwargs(project_guid=None, family_guid=None, individual_guid def get_json_for_samples(samples, **kwargs): - """Returns a JSON representation of the given list of Samples. - - Args: - samples (array): array of django models for the Samples. - Returns: - array: array of json objects - """ - return get_json_for_queryset(samples, **_get_sample_json_kwargs(**kwargs)) -def get_json_for_sample(sample, **kwargs): - """Returns a JSON representation of the given Sample. +def get_json_for_datasets(datasets, project_guid): + return get_json_for_queryset(datasets, additional_values={ + 'projectGuid': Value(project_guid), + 'activeIndividuals': ArrayAgg('active_individuals__guid', distinct=True, filter=Q(active_individuals__isnull=False)), + 'inactiveIndividuals': ArrayAgg('inactive_individuals__guid', distinct=True, filter=Q(inactive_individuals__isnull=False)), + }) - Args: - sample (object): Django model for the Sample. - Returns: - dict: json object - """ +def get_json_for_sample(sample, **kwargs): return _get_json_for_model(sample, **_get_sample_json_kwargs(**kwargs)) def get_json_for_analysis_groups(analysis_groups, project_guid=None, skip_nested=False, is_dynamic=False, **kwargs): - """Returns a JSON representation of the given list of AnalysisGroups. - - Args: - analysis_groups (array): array of django models for the AnalysisGroups. - project_guid (string): An optional field to use as the projectGuid instead of querying the DB - Returns: - array: array of json objects - """ - def _process_result(result, group): result.update({ 'familyGuids': [f.guid for f in group.families.all()] @@ -381,14 +319,6 @@ def _process_result(result, group): def get_json_for_analysis_group(analysis_group, **kwargs): - """Returns a JSON representation of the given AnalysisGroup. - - Args: - analysis_group (object): Django model for the AnalysisGroup. - project_guid (string): An optional field to use as the projectGuid instead of querying the DB - Returns: - dict: json object - """ return _get_json_for_model(analysis_group, get_json_for_models=get_json_for_analysis_groups, **kwargs) @@ -449,14 +379,18 @@ def get_json_for_saved_variants_child_entities(tag_cls, saved_variant_id_map, ta guid_key = 'tagGuid' nested_fields = None + additional_model_fields = ['id'] if tag_cls == VariantTag: nested_fields = [ {'fields': ('variant_tag_type', field), 'key': field} for field in ['name', 'category', 'color']] + related_searches = VariantSearchResults.objects.filter(search_hash=OuterRef('search_hash')).values('variant_search__name') + tag_models = tag_models.annotate(search_name=Subquery(related_searches[:1])) + additional_model_fields.append('search_name') elif tag_cls == VariantNote: guid_key = 'noteGuid' tags = get_json_for_queryset( - tag_models, guid_key=guid_key, nested_fields=nested_fields, additional_model_fields=['id']) + tag_models, guid_key=guid_key, nested_fields=nested_fields, additional_model_fields=additional_model_fields) if tag_cls == VariantFunctionalData: _format_functional_tags(tags) elif tag_cls == VariantTag: @@ -551,36 +485,14 @@ def get_json_for_discovery_tags(variants, user): def get_json_for_variant_note(note): - """Returns a JSON representation of the given variant note. - - Args: - note (object): Django model for the VariantNote. - Returns: - dict: json object - """ return _get_json_for_model(note, guid_key='noteGuid') def get_json_for_gene_notes(notes, user): - """Returns a JSON representation of the given gene note. - - Args: - note (object): Django model for the GeneNote. - Returns: - dict: json object - """ - return _get_json_for_models(notes, user=user, guid_key='noteGuid') def get_json_for_gene_notes_by_gene_id(gene_ids, user): - """Returns a JSON representation of the gene notes for the given gene ids. - - Args: - note (object): Django model for the GeneNote. - Returns: - dict: json object - """ notes_by_gene_id = defaultdict(list) for note in get_json_for_gene_notes(GeneNote.objects.filter(gene_id__in=gene_ids), user): notes_by_gene_id[note['geneId']].append(note) @@ -616,13 +528,6 @@ def get_json_for_locus_lists(locus_lists, user, include_metadata=True, additiona def get_json_for_locus_list(locus_list, user): - """Returns a JSON representation of the given LocusList. - - Args: - locus_list (object): LocusList django model. - Returns: - dict: json object - """ result = _get_json_for_model(locus_list, user=user) intervals = _get_json_for_models(locus_list.locuslistinterval_set.all()) @@ -660,7 +565,6 @@ def get_json_for_project_collaborator_groups(project): def get_json_for_project_collaborator_list(user, project): - """Returns a JSON representation of the collaborators in the given project""" collaborator_list = list( get_project_collaborators_by_username( user, project, fields=['username', 'email', 'display_name'], include_permissions=True, @@ -671,7 +575,6 @@ def get_json_for_project_collaborator_list(user, project): def get_project_collaborators_by_username(user, project, fields, include_permissions=False, expand_user_groups=False): - """Returns a JSON representation of the collaborators in the given project""" collaborators = {} if not anvil_enabled(): if expand_user_groups: @@ -785,6 +688,7 @@ def get_json_for_rna_seq_outliers(filters, significant_only=True, individual_gui models, nested_fields=[ {'fields': ('sample', 'tissue_type'), 'key': 'tissueType'}, + {'fields': ('sample', 'sequencing_type'), 'key': 'sequencingType'}, {'fields': ('sample', 'individual', 'guid'), 'key': 'individualGuid', 'value': individual_guid}, ], additional_values={'isSignificant': Value(True)} if significant_only else { diff --git a/seqr/views/utils/orm_to_json_utils_tests.py b/seqr/views/utils/orm_to_json_utils_tests.py index f5a68c23a0..8f344bb717 100644 --- a/seqr/views/utils/orm_to_json_utils_tests.py +++ b/seqr/views/utils/orm_to_json_utils_tests.py @@ -1,12 +1,12 @@ from django.contrib.auth.models import User import mock from copy import deepcopy -from seqr.models import Project, Sample, IgvSample, SavedVariant, VariantNote, LocusList, VariantSearch +from seqr.models import Project, IgvSample, SavedVariant, VariantNote, LocusList, VariantSearch from seqr.views.utils.orm_to_json_utils import get_json_for_user, _get_json_for_project, \ get_json_for_sample, get_json_for_variant_note, get_json_for_locus_list, \ get_json_for_saved_searches, get_json_for_saved_variants_with_tags, get_json_for_current_user from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, \ - PROJECT_FIELDS, SAMPLE_FIELDS, SAVED_VARIANT_FIELDS, \ + PROJECT_FIELDS, SAVED_VARIANT_FIELDS, \ FUNCTIONAL_FIELDS, SAVED_SEARCH_FIELDS, LOCUS_LIST_DETAIL_FIELDS, PA_LOCUS_LIST_FIELDS, IGV_SAMPLE_FIELDS, \ TAG_FIELDS, VARIANT_NOTE_FIELDS @@ -95,12 +95,6 @@ def test_json_for_project(self): self.assertSetEqual(set(json.keys()), PROJECT_FIELDS) - def test_json_for_sample(self): - sample = Sample.objects.first() - json = get_json_for_sample(sample) - - self.assertSetEqual(set(json.keys()), SAMPLE_FIELDS) - def test_json_for_igv_sample(self): sample = IgvSample.objects.first() json = get_json_for_sample(sample) diff --git a/seqr/views/utils/pedigree_info_utils.py b/seqr/views/utils/pedigree_info_utils.py index f29f8a2cc1..6fd9c94f36 100644 --- a/seqr/views/utils/pedigree_info_utils.py +++ b/seqr/views/utils/pedigree_info_utils.py @@ -1,4 +1,3 @@ -"""Utilities for parsing .fam files or other tables that describe individual pedigree structure.""" import difflib import os import json @@ -15,7 +14,7 @@ from seqr.utils.middleware import ErrorsWarningsException from seqr.views.utils.json_utils import _to_snake_case, _to_title_case from seqr.views.utils.permissions_utils import user_is_pm, get_pm_user_emails -from seqr.models import Individual, Sample +from seqr.models import Individual logger = SeqrLogger(__name__) @@ -25,23 +24,6 @@ def parse_pedigree_table(parsed_file, filename, user, project): - """Validates and parses pedigree information from a .fam, .tsv, or Excel file. - - Args: - parsed_file (array): The parsed output from the raw file. - filename (string): The original filename - used to determine the file format based on the suffix. - user (User): Django User object - project (Project): Django Project object - - Return: - A 3-tuple that contains: - ( - json_records (list): list of dictionaries, with each dictionary containing info about - one of the individuals in the input data - errors (list): list of error message strings - warnings (list): list of warning message strings - ) - """ header_string = str(parsed_file[0]) is_merged_pedigree_sample_manifest = "do not modify" in header_string.lower() and "Broad" in header_string if is_merged_pedigree_sample_manifest: @@ -153,32 +135,6 @@ def parse_hpo_terms(hpo_term_string): def _convert_fam_file_rows_to_json(column_map, rows, required_columns=None, update_features=False): - """Parse the values in rows and convert them to a json representation. - - Args: - rows (list): a list of rows where each row is a list of strings corresponding to values in the table - - Returns: - list: a list of dictionaries with each dictionary being a json representation of a parsed row. - For example: - { - 'familyId': family_id, - 'individualId': individual_id, - 'paternalId': paternal_id, - 'maternalId': maternal_id, - 'sex': sex, - 'affected': affected, - 'notes': notes, - 'codedPhenotype': , - 'hpoTermsPresent': [...], - 'hpoTermsAbsent': [...], - 'fundingSource': [...], - 'caseReviewStatus': [...], - } - - Raises: - ValueError: if there are unexpected values or row sizes - """ required_columns = [JsonConstants.FAMILY_ID_COLUMN, JsonConstants.INDIVIDUAL_ID_COLUMN] + (required_columns or []) missing_cols = [_to_title_case(_to_snake_case(col)) for col in set(required_columns) - set(column_map.values())] if update_features and JsonConstants.FEATURES not in column_map.values(): @@ -252,19 +208,6 @@ def _format_value(value, column): def validate_fam_file_records(project, records, errors=None, clear_invalid_values=False, update_features=False, related_guids=None, search_dataset_type=None, validate_expected_samples=None): - """Basic validation such as checking that parents have the same family id as the child, etc. - - Args: - records (list): a list of dictionaries (see return value of #process_rows). - - Returns: - dict: json representation of any errors, warnings, or info messages: - { - 'errors': ['error text1', 'error text2', ...], - 'warnings': ['warning text1', 'warning text2', ...], - 'info': ['info message', ...], - } - """ records_by_id = {r[JsonConstants.PREVIOUS_INDIVIDUAL_ID_COLUMN]: r for r in records if r.get(JsonConstants.PREVIOUS_INDIVIDUAL_ID_COLUMN)} records_by_id.update({r[JsonConstants.INDIVIDUAL_ID_COLUMN]: r for r in records}) @@ -361,22 +304,24 @@ def get_validated_related_individuals(project, records_by_id, errors, related_gu affected = records_by_id[individual_id].get(JsonConstants.AFFECTED_COLUMN, Individual.AFFECTED_STATUS_UNKNOWN) affected_status_by_family[family_id].append(affected) - search_samples = Sample.objects.filter(individual__family__project=project, is_active=True) + search_individuals = Individual.objects.filter(family__project=project) sample_type = None if search_dataset_type: - search_samples = search_samples.filter(dataset_type=search_dataset_type) + search_individuals = search_individuals.filter(active_datasets__dataset_type=search_dataset_type) if search_sample_type: - search_samples = search_samples.filter(sample_type=search_sample_type) - elif search_samples: - sample_type = search_samples.first().sample_type + search_individuals = search_individuals.filter(active_datasets__sample_type=search_sample_type) + elif search_individuals: + sample_type = search_individuals.first().active_datasets.first().sample_type + else: + search_individuals = search_individuals.filter(active_datasets__dataset_type__isnull=False) previous_loaded_individuals = { i[JsonConstants.INDIVIDUAL_ID_COLUMN]: i - for i in search_samples.values( - 'individual_id', **{ - JsonConstants.INDIVIDUAL_ID_COLUMN: F('individual__individual_id'), - JsonConstants.FAMILY_ID_COLUMN: F('individual__family__family_id'), - }) + for i in search_individuals.values( + **{ + JsonConstants.INDIVIDUAL_ID_COLUMN: F('individual_id'), + JsonConstants.FAMILY_ID_COLUMN: F('family__family_id'), + }).annotate(individual_id=F('id')) } if validate_expected_samples: @@ -450,13 +395,6 @@ def _validate_parent(row, parent_id_type, parent_id_field, parent_guid_field, ex def _is_header_row(row): - """Checks if the 1st row of a table is a header row - - Args: - row (string): 1st row of a table - Returns: - True if it's a header row rather than data - """ row = row.lower() if "family" in row and ("indiv" in row or "participant" in row): return True @@ -486,18 +424,6 @@ def _parse_merged_pedigree_sample_manifest_rows(rows): def _parse_merged_pedigree_sample_manifest_format(rows, project): - """Does post-processing of rows from Broad's sample manifest + pedigree table format. Expected columns are: - - Kit ID, Well Position, Sample ID, Family ID, Collaborator Participant ID, Collaborator Sample ID, - Paternal Sample ID, Maternal ID, Gender, Affected Status, Volume, Concentration, Notes, Coded Phenotype, - Data Use Restrictions - - Args: - rows (list): A list of lists where each list contains values from each column in the table. - - Returns: - 3-tuple: rows, sample_manifest_rows, kit_id - """ c = MergedPedigreeSampleManifestConstants kit_id = rows[0][c.KIT_ID_COLUMN] diff --git a/seqr/views/utils/permissions_utils.py b/seqr/views/utils/permissions_utils.py index 5ec013e8f9..d5a03f97cd 100644 --- a/seqr/views/utils/permissions_utils.py +++ b/seqr/views/utils/permissions_utils.py @@ -112,14 +112,6 @@ def get_internal_projects(): def get_project_and_check_permissions(project_guid, user, **kwargs): - """Retrieves Project with the given guid after checking that the given user has permission to - retrieve the given project. - - Args: - project_guid (string): GUID of project to retrieve - user (User): Django User object - can_edit (bool): If user need edit permission - """ return _get_project_and_check_permissions(project_guid, user, check_project_permissions, **kwargs) def get_project_and_check_pm_permissions(project_guid, user, override_permission_func=None): diff --git a/seqr/views/utils/terra_api_utils.py b/seqr/views/utils/terra_api_utils.py index 05c9f3ae5c..06fff506d0 100644 --- a/seqr/views/utils/terra_api_utils.py +++ b/seqr/views/utils/terra_api_utils.py @@ -1,5 +1,3 @@ -"""Provide python bindings for the AnVIL Terra API.""" - from datetime import datetime import google.auth.transport.requests import json @@ -28,33 +26,17 @@ class TerraAPIException(Exception): def __init__(self, message, status_code): - """ - Custom Exception to capture Terra API call failures - - :param message: error message - :param status_code: the status code associated with the failed request - """ super(TerraAPIException, self).__init__(message) self.status_code = status_code class TerraNotFoundException(TerraAPIException): def __init__(self, message): - """ - Custom Exception to capture Terra API calls that fail with 404 Not Found - - :param message: error message - """ super(TerraNotFoundException, self).__init__(message, 404) class TerraRefreshTokenFailedException(TerraAPIException): def __init__(self, message): - """ - Custom Exception to capture refresh token failures. Maps to 401 error to redirect user to login - - :param message: error message - """ super(TerraRefreshTokenFailedException, self).__init__(message, 401) @@ -178,15 +160,6 @@ def _user_anvil_call(method, path, user, **kwargs): def list_anvil_workspaces(user): - """Get all the workspaces accessible by the logged-in user. - - :param - user (User model): who's credentials will be used to access AnVIL - :return - A list of workspaces that the user has access (OWNER, WRITER, or READER). Each of the workspace has - its name and namespace. - - """ return _user_anvil_call( 'get', 'api/workspaces?fields=public,workspace.name,workspace.namespace', user, cache_time=TERRA_WORKSPACE_CACHE_EXPIRE_SECONDS, diff --git a/seqr/views/utils/terra_api_utils_tests.py b/seqr/views/utils/terra_api_utils_tests.py index 8268bd06ac..f416dc7393 100644 --- a/seqr/views/utils/terra_api_utils_tests.py +++ b/seqr/views/utils/terra_api_utils_tests.py @@ -13,7 +13,7 @@ add_service_account, has_service_account_access, get_anvil_group_members, user_get_anvil_groups GET_WORKSPACE_PATH = 'api/workspaces?fields=public,workspace.name,workspace.namespace' -AUTH_EXTRA_DATA = {"expires": 3599, "auth_time": TOKEN_AUTH_TIME, "token_type": "Bearer", "access_token": "ya29.EXAMPLE"} +AUTH_EXTRA_DATA = {"expires": 3599, "auth_time": TOKEN_AUTH_TIME, "token_type": "Bearer", "access_token": "ya29.EXAMPLE"} # nosec LIST_WORKSPACE_RESPONSE = '[{"accessLevel": "PROJECT_OWNER", "public": false, "workspace": {"attributes": {"description": "Workspace for seqr project"}, "authorizationDomain": [], "bucketName": "fc-237998e6-663d-40b9-bd13-57c3bb6ac593", "createdBy": "test1@test.com", "createdDate": "2020-09-09T15:10:32.816Z", "isLocked": false, "lastModified": "2020-09-09T15:10:32.818Z", "name": "1000 Genomes Demo", "namespace": "my-seqr-billing", "workflowCollectionName": "237998e6-663d-40b9-bd13-57c3bb6ac593", "workspaceId": "237998e6-663d-40b9-bd13-57c3bb6ac593" }, "workspaceSubmissionStats": {"runningSubmissionsCount": 0}},\ {"accessLevel": "READER","public": true, "workspace": {"attributes": {"tag:tags": {"itemsType": "AttributeValue","items": ["differential-expression","tutorial"]},"description": "[DEGenome](https://github.com/eweitz/degenome) transforms differential expression data into inputs for [exploratory genome analysis with Ideogram.js](https://eweitz.github.io/ideogram/differential-expression?annots-url=https://www.googleapis.com/storage/v1/b/degenome/o/GLDS-4_array_differential_expression_ideogram_annots.json). \\n\\nTry the [Notebook tutorial](https://app.terra.bio/#workspaces/degenome/degenome/notebooks/launch/degenome-tutorial.ipynb), where you can step through using DEGenome to analyze expression for mice flown in space!"},"authorizationDomain": [],"bucketName": "fc-2706d493-5fce-4fb2-9993-457c30364a06","createdBy": "test2@test.com","createdDate": "2020-01-14T10:21:14.575Z","isLocked": false,"lastModified": "2020-02-01T13:28:27.309Z","name": "degenome","namespace": "degenome","workflowCollectionName": "2706d493-5fce-4fb2-9993-457c30364a06","workspaceId": "2706d493-5fce-4fb2-9993-457c30364a06"},"workspaceSubmissionStats": {"runningSubmissionsCount": 0}},\ {"accessLevel": "PROJECT_OWNER","public": false, "workspace": {"attributes": {"description": "A workspace for seqr project"},"authorizationDomain": [],"bucketName": "fc-6a048145-c134-4004-a009-42824f826ee8","createdBy": "test3@test.com","createdDate": "2020-09-09T15:12:30.142Z","isLocked": false,"lastModified": "2020-09-09T15:12:30.145Z","name": "seqr-project 1000 Genomes Demo","namespace": "my-seqr-billing","workflowCollectionName": "6a048145-c134-4004-a009-42824f826ee8","workspaceId": "6a048145-c134-4004-a009-42824f826ee8"},"workspaceSubmissionStats": {"runningSubmissionsCount": 0}}]' diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index a8af246857..4b8b8a6334 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -14,7 +14,7 @@ import responses from urllib.parse import quote_plus, urlparse -from seqr.models import Project, SavedVariant, CAN_VIEW, CAN_EDIT +from seqr.models import Project, CAN_VIEW, CAN_EDIT WINDOW_REGEX_TEMPLATE = 'window\.{key}=(?P[^)<]+)' @@ -27,7 +27,7 @@ def _databases_support_transactions(cls): @classmethod def _rollback_atomics(cls, atomics): - """Django testcases asssume either all database support transactions or none do. This properly cleans up transaction blocks on a per-db basis""" + # Django testcases asssume either all database support transactions or none do. This properly cleans up transaction blocks on a per-db basis for db_name in reversed(cls._databases_names()): if connections[db_name].features.supports_transactions: transaction.set_rollback(True, using=db_name) @@ -170,15 +170,6 @@ def login_data_manager_user(self): def _check_login(self, url, permission_level, request_data=None, login_redirect_url='/api/login-required-error', policy_redirect_url='/api/policy-required-error', permission_denied_error=403): - """For integration tests of django views that can only be accessed by a logged-in user, - the 1st step is to authenticate. This function checks that the given url redirects requests - if the user isn't logged-in, and then authenticates a test user. - - Args: - test_case (object): the django.TestCase or unittest.TestCase object - url (string): The url of the django view being tested. - permission_level (string): what level of permission this url requires - """ # check that it redirects if you don't login parsed_url = urlparse(url) next_query = quote_plus('?{}'.format(parsed_url.query)) if parsed_url.query else '' @@ -703,9 +694,8 @@ def _get_list_param(call, param): INTERNAL_INDIVIDUAL_FIELDS = deepcopy(NO_INTERNAL_CASE_REVIEW_INDIVIDUAL_FIELDS) INTERNAL_INDIVIDUAL_FIELDS.update(CORE_INTERNAL_INDIVIDUAL_FIELDS) -SAMPLE_FIELDS = { - 'projectGuid', 'familyGuid', 'individualGuid', 'sampleGuid', 'createdDate', 'sampleType', 'sampleId', 'isActive', - 'loadedDate', 'datasetType', +DATASET_FIELDS = { + 'projectGuid', 'datasetGuid', 'sampleType', 'loadedDate', 'datasetType', 'activeIndividuals', 'inactiveIndividuals', } IGV_SAMPLE_FIELDS = { @@ -722,7 +712,7 @@ def _get_list_param(call, param): SAVED_VARIANT_DETAIL_FIELDS.update(SAVED_VARIANT_FIELDS) TAG_FIELDS = { - 'tagGuid', 'name', 'category', 'color', 'searchHash', 'metadata', 'lastModifiedDate', 'createdBy', 'variantGuids', + 'tagGuid', 'name', 'category', 'color', 'searchHash', 'searchName', 'metadata', 'lastModifiedDate', 'createdBy', 'variantGuids', } VARIANT_NOTE_FIELDS = {'noteGuid', 'note', 'report', 'lastModifiedDate', 'createdBy', 'variantGuids'} diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index eab4d543b2..19088364e6 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -12,9 +12,9 @@ from matchmaker.models import MatchmakerSubmissionGenes, MatchmakerSubmission from reference_data.models import TranscriptInfo, Omim, GENOME_VERSION_GRCh38 from seqr.models import SavedVariant, VariantSearchResults, Family, LocusList, LocusListInterval, LocusListGene, \ - RnaSeqTpm, PhenotypePrioritization, Project, Sample, RnaSample, VariantTag, VariantTagType -from seqr.utils.search.utils import variant_dataset_type + RnaSeqTpm, PhenotypePrioritization, Project, Dataset, RnaSample, VariantTag, VariantTagType from seqr.utils.gene_utils import get_genes_for_variants +from seqr.utils.xpos_utils import parse_variant_id from seqr.views.utils.json_to_orm_utils import create_model_from_json from seqr.views.utils.orm_to_json_utils import get_json_for_discovery_tags, get_json_for_locus_lists, \ get_json_for_queryset, get_json_for_rna_seq_outliers, get_json_for_saved_variants_with_tags, \ @@ -63,6 +63,13 @@ def parse_saved_variant_json(variant_json, family_id): } +def variant_dataset_type(variant): + if not parse_variant_id(variant['variantId']): + sample_type = Dataset.SAMPLE_TYPE_WGS if 'endChrom' in variant else Dataset.SAMPLE_TYPE_WES + return f'{Dataset.DATASET_TYPE_SV_CALLS}_{sample_type}' + return Dataset.DATASET_TYPE_MITO_CALLS if 'mitomapPathogenic' in variant else Dataset.DATASET_TYPE_VARIANT_CALLS + + def _transcript_sort(gene_id, saved_variant_json, main_transcript_id): gene_transcripts = saved_variant_json['transcripts'][gene_id] is_main_gene = bool(main_transcript_id) and any(t.get('transcriptId') == main_transcript_id for t in gene_transcripts) @@ -171,7 +178,7 @@ def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], comp_het_ def get_saved_variant_annotations(variant_keys: abc.Iterable[tuple[int, str]], genome_version: str, primary_id_field: str = 'variant_id', group_by_field=None, dataset_type: str = None) -> dict[str, dict]: - dataset_type = dataset_type or Sample.DATASET_TYPE_VARIANT_CALLS + dataset_type = dataset_type or Dataset.DATASET_TYPE_VARIANT_CALLS variant_ids = {variant_id for _, variant_id in variant_keys} keys = None if primary_id_field == 'key': @@ -490,9 +497,9 @@ def _set_response_gene_scores(response, family_genes, gene_ids): def _add_sample_count_stats(response, genome_versions): - sample_counts = Sample.objects.filter( - is_active=True, individual__family__project__is_demo=False, individual__family__project__genome_version__in=genome_versions, - ).values('sample_type', 'dataset_type').annotate(count=Count('*')) + sample_counts = Dataset.objects.filter( + active_individuals__family__project__is_demo=False, active_individuals__family__project__genome_version__in=genome_versions, + ).values('sample_type', 'dataset_type').annotate(count=Count('active_individuals', distinct=True)) counts_by_dataset_type = defaultdict(dict) for sample_type, dataset_type, count in sample_counts.values_list('sample_type', 'dataset_type', 'count'): counts_by_dataset_type[dataset_type][sample_type] = count diff --git a/settings.py b/settings.py index 94fc51a8b3..5c3871d8cf 100644 --- a/settings.py +++ b/settings.py @@ -4,8 +4,6 @@ import string import subprocess # nosec -from ssl import create_default_context - import google.auth import google.auth.transport.requests diff --git a/ui/pages/Project/components/EditDatasetsButton.jsx b/ui/pages/Project/components/EditDatasetsButton.jsx index 90e5e2fb61..23b8791897 100644 --- a/ui/pages/Project/components/EditDatasetsButton.jsx +++ b/ui/pages/Project/components/EditDatasetsButton.jsx @@ -11,7 +11,7 @@ import { UPLOAD_PROJECT_IGV_FIELD } from 'shared/components/form/IGVUploadField' import FileUploadField from 'shared/components/form/XHRUploaderField' import { Select } from 'shared/components/form/Inputs' import AddWorkspaceDataForm from 'shared/components/panel/LoadWorkspaceDataForm' -import { LOAD_RNA_FIELDS, TISSUE_DISPLAY } from 'shared/utils/constants' +import { LOAD_RNA_FIELDS, TISSUE_DISPLAY, PRODUCT_DISPLAY } from 'shared/utils/constants' import { addIGVDataset, uploadRnaSeq } from '../reducers' import { getCurrentProject, getProjectGuid, getRnaSeqUploadStats } from '../selectors' @@ -72,6 +72,13 @@ const PROJECT_LOAD_RNA_FIELDS = [ options: Object.entries(TISSUE_DISPLAY).map(([value, name]) => ({ value, name })), validate: validators.required, }, + { + name: 'sequencingType', + label: 'Sequencing Type', + component: Select, + options: Object.entries(PRODUCT_DISPLAY).map(([value, name]) => ({ value, name })), + validate: validators.required, + }, ...LOAD_RNA_FIELDS.slice(-2), ] diff --git a/ui/pages/Project/components/ProjectOverview.jsx b/ui/pages/Project/components/ProjectOverview.jsx index 82ef203dad..be47568655 100644 --- a/ui/pages/Project/components/ProjectOverview.jsx +++ b/ui/pages/Project/components/ProjectOverview.jsx @@ -29,6 +29,7 @@ import { getProjectAnalysisGroupFamilySizeHistogram, getProjectAnalysisGroupDataLoadedFamilySizeHistogram, getProjectAnalysisGroupSamplesByTypes, + getProjectAnalysisGroupRnaSamplesByTypes, getProjectAnalysisGroupMmeSubmissionDetails, getMmeSubmissionsLoading, } from '../selectors' @@ -303,15 +304,19 @@ class DatasetSection extends React.PureComponent { } -const Dataset = React.memo(({ showLoadWorkspaceData, hasAnvil, samplesByType, user }) => { - const datasetSections = samplesByType.map(([sampleTypeKey, loadedSampleCounts]) => { +const Dataset = React.memo(({ showLoadWorkspaceData, hasAnvil, samplesByType, rnaSamplesByType, user }) => { + const datasetSections = [...Object.entries(samplesByType).map(([sampleTypeKey, loadedSampleCounts]) => { const [sampleType, datasetType] = sampleTypeKey.split('__') return { key: sampleTypeKey, title: `${SAMPLE_TYPE_LOOKUP[sampleType] || sampleType}${DATASET_TITLE_LOOKUP[datasetType] || ''} Datasets`, content: , } - }).sort((a, b) => a.title.localeCompare(b.title)) + }), ...rnaSamplesByType.map(([dataType, loadedSampleCounts]) => ({ + key: dataType, + title: `RNA${DATASET_TITLE_LOOKUP[dataType]} Datasets`, + content: , + }))].sort((a, b) => a.title.localeCompare(b.title)) const noLoadedData = !datasetSections.length if (noLoadedData) { @@ -358,6 +363,7 @@ const Dataset = React.memo(({ showLoadWorkspaceData, hasAnvil, samplesByType, us Dataset.propTypes = { samplesByType: PropTypes.object.isRequired, + rnaSamplesByType: PropTypes.arrayOf(PropTypes.any).isRequired, hasAnvil: PropTypes.bool, showLoadWorkspaceData: PropTypes.bool, user: PropTypes.object.isRequired, @@ -366,6 +372,7 @@ Dataset.propTypes = { const mapDatasetStateToProps = (state, ownProps) => ({ user: getUser(state), samplesByType: getProjectAnalysisGroupSamplesByTypes(state, ownProps), + rnaSamplesByType: getProjectAnalysisGroupRnaSamplesByTypes(state, ownProps), }) const DatasetOverview = connect(mapDatasetStateToProps)(Dataset) diff --git a/ui/pages/Project/components/RnaSeqResultPage.jsx b/ui/pages/Project/components/RnaSeqResultPage.jsx index 8477420bb2..0546c4916c 100644 --- a/ui/pages/Project/components/RnaSeqResultPage.jsx +++ b/ui/pages/Project/components/RnaSeqResultPage.jsx @@ -5,7 +5,7 @@ import { connect } from 'react-redux' import { Loader, Grid, Dropdown } from 'semantic-ui-react' import { getGenesById, getIndividualsByGuid, getRnaSeqSignificantJunctionData } from 'redux/selectors' -import { RNASEQ_JUNCTION_PADDING, TISSUE_DISPLAY } from 'shared/utils/constants' +import { RNASEQ_JUNCTION_PADDING } from 'shared/utils/constants' import DataLoader from 'shared/components/DataLoader' import FamilyReads from 'shared/components/panel/family/FamilyReads' import RnaSeqJunctionOutliersTable from 'shared/components/table/RnaSeqJunctionOutliersTable' @@ -46,44 +46,44 @@ class BaseRnaSeqResultPage extends React.PureComponent { } state = { - tissueType: '', + selectedDataType: null, } onTissueChange = (e, data) => { - this.setState({ tissueType: data.value }) + this.setState({ selectedDataType: data.value }) } render() { const { familyGuid, rnaSeqData, significantJunctionOutliers, genesById, tissueOptions } = this.props - const { tissueType } = this.state - const defaultTissue = tissueOptions?.length > 0 ? tissueOptions[0].value : null - const showTissueType = tissueType === '' ? defaultTissue : tissueType - const tissueDisplay = TISSUE_DISPLAY[showTissueType] + const { selectedDataType } = this.state + const showDataType = selectedDataType || tissueOptions[0].value + const [showTissueType, showSequencingType] = showDataType.split('-') const outlierPlotConfigs = OUTLIER_VOLCANO_PLOT_CONFIGS.map((config) => { - const data = rnaSeqData[config.key] - return ({ data: data.filter(outlier => outlier.tissueType === showTissueType), ...config }) + const data = rnaSeqData[config.key].filter( + outlier => outlier.tissueType === showTissueType && outlier.sequencingType === showSequencingType, + ) + return ({ data, ...config }) }).filter(({ data }) => data.length) const tableData = significantJunctionOutliers.reduce( - (acc, outlier) => (outlier.tissueType === showTissueType ? [...acc, outlier] : acc), [], + (acc, outlier) => ( + (outlier.tissueType === showTissueType && outlier.sequencingType === showSequencingType) ? + [...acc, outlier] : acc + ), [], ) return (
- {(showTissueType || tissueOptions?.length > 1) && ( - - Tissue type:   - {tissueOptions?.length > 1 ? ( - - ) : tissueDisplay} - - )} + + {tissueOptions?.length > 1 ? ( + + ) : tissueOptions[0].text} + { outlierPlotConfigs.length > 0 && ( }> diff --git a/ui/pages/Project/fixtures.js b/ui/pages/Project/fixtures.js index 033fc74b57..6689bcb1e0 100644 --- a/ui/pages/Project/fixtures.js +++ b/ui/pages/Project/fixtures.js @@ -121,7 +121,7 @@ export const STATE1 = { sex: 'M', }, }, - samplesByGuid: {}, + datasetsByGuid: {}, mmeSubmissionsByGuid: {}, project: { createdDate: '2016-05-16T05:37:08.634Z', @@ -310,19 +310,14 @@ export const STATE_WITH_2_FAMILIES = { sex: 'M', }, }, - samplesByGuid: { + datasetsByGuid: { S2310656_wal_mc16200_mc16203: { - createdDate: "2018-03-30T11:50:40.079Z", - datasetFilePath: "gs://seqr-datasets/GRCh37/cmg_sankaran_wes/CMG_MYOSEQ.vcf.gz", - datasetName: null, datasetType: "SNV_INDEL", - familyGuid: 'F011652_2', - individualGuid: "I021476_na19678_2", + activeIndividuals: ["I021476_na19678_2"], + inactiveIndividuals: [], loadedDate: "2018-03-13T13:25:21.551Z", projectGuid: "R0237_1000_genomes_demo", - sampleGuid: "S2310656_wal_mc16200_mc16203", - sampleId: "WAL_MC16200_MC16203", - isActive: true, + datasetGuid: "S2310656_wal_mc16200_mc16203", sampleType: "WES", }, }, diff --git a/ui/pages/Project/reducers.js b/ui/pages/Project/reducers.js index c2181b8dfe..9d0d355724 100644 --- a/ui/pages/Project/reducers.js +++ b/ui/pages/Project/reducers.js @@ -56,11 +56,8 @@ export const loadMmeSubmissions = () => loadCurrentProjectChildEntities('mme sub const loadFamilyNotes = () => loadCurrentProjectChildEntities('family notes', REQUEST_FAMILIES, RECEIVE_FAMILIES) -const loadSamples = () => loadCurrentProjectChildEntities('samples', REQUEST_INDIVIDUALS) - export const loadProjectExportData = () => (dispatch, getState) => Promise.all([ loadIndividuals()(dispatch, getState), - loadSamples()(dispatch, getState), loadFamilyNotes()(dispatch, getState), ]) diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index 7b5b560511..70a132444a 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -12,9 +12,11 @@ import { INDIVIDUAL_HAS_DATA_FIELD, MME_TAG_NAME, TISSUE_DISPLAY, + PRODUCT_DISPLAY, SIMPLIFIED_SEX_LOOKUP, } from 'shared/utils/constants' import { toCamelcase, toSnakecase, snakecaseToTitlecase } from 'shared/utils/stringUtils' +import { compareObjects } from 'shared/utils/sortUtils' import { getProjectsByGuid, getFamiliesGroupedByProjectGuid, getIndividualsByGuid, getGenesById, getUser, @@ -23,7 +25,7 @@ import { getVariantTagsByGuid, getUserOptionsByUsername, getNotesByFamilyType, getVariantTagNotesByFamilyVariants, getPhenotypeGeneScoresByIndividual, getActiveDatasetsByIndividual, getRnaSeqDataByIndividual, familyPassesFilters, getAnalysisGroupGuid, getCurrentAnalysisGroupFamilyGuids, - getDatasetsByIndividual, getActiveDatasetsByFamily, getMinMaxDatasetsByFamily, + getDatasetsByIndividual, getActiveDatasetsByFamily, getMinMaxDatasetsByFamily, getDatasetsGroupedByProjectGuid, } from 'redux/selectors' import { @@ -170,9 +172,32 @@ export const getProjectAnalysisGroupIndividualsByGuid = createSelector( ) export const getProjectAnalysisGroupSamplesByTypes = createSelector( + getProjectGuid, + getCurrentAnalysisGroupFamilyGuids, + getProjectAnalysisGroupIndividualsByGuid, + getDatasetsGroupedByProjectGuid, + (projectGuid, analysisGroupFamilyGuids, analysisGroupIndividuals, datasetsByProject) => Object.values( + datasetsByProject[projectGuid] || {}, + ).sort(compareObjects('loadedDate')).reduce((acc, { sampleType, datasetType, loadedDate, activeIndividuals, inactiveIndividuals }) => { + const key = `${sampleType}__${datasetType}` + if (!acc[key]) { + acc[key] = [] + } + const individualGuids = [...(activeIndividuals || []), ...(inactiveIndividuals || [])] + const count = !analysisGroupFamilyGuids ? individualGuids.length : individualGuids.filter( + individualGuid => individualGuid in analysisGroupIndividuals, + ).length + if (count) { + acc[key].push({ loadedDate, count }) + } + return acc + }, {}), +) + +export const getProjectAnalysisGroupRnaSamplesByTypes = createSelector( getCurrentProject, getCurrentAnalysisGroupFamilyGuids, - (project, analysisGroupFamilyGuids) => Object.entries(project.sampleCounts || {}).map( + (project, analysisGroupFamilyGuids) => Object.entries(project.rnaSampleCounts || {}).map( ([key, typeCounts]) => ([key, typeCounts.map(({ familyCounts, ...data }) => ({ ...data, count: Object.entries(familyCounts).reduce((total, [familyGuid, count]) => ( @@ -831,17 +856,19 @@ export const getTissueOptionsByIndividualGuid = createSelector( const tissueTypesByIndividualGuid = Object.entries(rnaSeqDataByIndividualGuid || {}).map( ([individualGuid, rnaSeqData]) => ([ individualGuid, - [...new Set(Object.values(rnaSeqData || {}).map(Object.values).flat(2).map(({ tissueType }) => tissueType))], + [...new Set(Object.values(rnaSeqData || {}).map(Object.values).flat(2).map( + ({ tissueType, sequencingType }) => `${tissueType}-${sequencingType}`, + ))], ]), ) - return tissueTypesByIndividualGuid.reduce((acc, [individualGuid, tissueTypes]) => ( - tissueTypes.length > 0 ? { - ...acc, - [individualGuid]: tissueTypes.map(tissueType => ( - { key: tissueType, text: TISSUE_DISPLAY[tissueType] || 'Unknown Tissue', value: tissueType } - )), - } : acc - ), {}) + return tissueTypesByIndividualGuid.reduce((acc, [individualGuid, dataTypes]) => ({ + ...acc, + [individualGuid]: dataTypes.map(dataType => ({ + key: dataType, + value: dataType, + text: `Tissue type: ${TISSUE_DISPLAY[dataType.split('-')[0]] || 'Unknown Tissue'}, Sequencing Product: ${PRODUCT_DISPLAY[dataType.split('-')[1]] || 'Unknown Product'}`, + })), + }), {}) }, ) diff --git a/ui/pages/Search/components/VariantSearchResults.jsx b/ui/pages/Search/components/VariantSearchResults.jsx index ab86291822..8ce1167fe0 100644 --- a/ui/pages/Search/components/VariantSearchResults.jsx +++ b/ui/pages/Search/components/VariantSearchResults.jsx @@ -43,10 +43,10 @@ const LargeRow = styled(Grid.Row)` const scrollToTop = () => window.scrollTo(0, 0) -const DisplayVariants = React.memo(({ displayVariants, compoundHetToggle }) => ( +const DisplayVariants = React.memo(({ displayVariants, ...props }) => ( - + )) @@ -95,7 +95,7 @@ const BaseVariantSearchResultsContent = React.memo(({ , - , + , diff --git a/ui/pages/Search/fixtures.js b/ui/pages/Search/fixtures.js index 49bbe7a4f2..013130e72e 100644 --- a/ui/pages/Search/fixtures.js +++ b/ui/pages/Search/fixtures.js @@ -158,59 +158,33 @@ export const STATE = { paternalId: '', }, }, - samplesByGuid: { + datasetsByGuid: { S2310658_wal_mc16200_mc16203: { - createdDate: "2018-03-30T11:50:40.079Z", - elasticsearchIndex: "CMG_MYOSEQ", datasetType: "SNV_INDEL", - individualGuid: "I021476_na19678", + activeIndividuals: ["I021476_na19678"], + inactiveIndividuals: [], loadedDate: "2018-03-13T13:25:21.551Z", projectGuid: PROJECT_GUID, - familyGuid: FAMILY_GUID, - sampleGuid: "S2310656_wal_mc16200_mc16203", - sampleId: "WAL_MC16200_MC16203", - isActive: true, + datasetGuid: "S2310658_wal_mc16200_mc16203", sampleType: "WES", }, S2310657_wal_mc16200_mc16203: { - createdDate: "2018-03-30T11:50:40.079Z", - elasticsearchIndex: "CMG_MYOSEQ", datasetType: "SV", - individualGuid: "I021476_na19678", + activeIndividuals: ["I021476_na19678"], + inactiveIndividuals: null, loadedDate: "2018-03-13T13:25:21.551Z", projectGuid: PROJECT_GUID, - sampleGuid: "S2310656_wal_mc16200_mc16203", - sampleId: "WAL_MC16200_MC16203", - isActive: true, + datasetGuid: "S2310657_wal_mc16200_mc16203", sampleType: "WES", }, S2310656_wal_mc16200_mc16203: { - createdDate: "2018-03-30T11:50:40.079Z", - elasticsearchIndex: "CMG_MYOSEQ", datasetType: "SNV_INDEL", - individualGuid: "I021476_na19678", + inactiveIndividuals: ["I021476_na19678"], loadedDate: "2018-03-13T13:25:21.551Z", projectGuid: PROJECT_GUID, - familyGuid: FAMILY_GUID, - sampleGuid: "S2310656_wal_mc16200_mc16203", - sampleId: "WAL_MC16200_MC16203", - isActive: false, + datasetGuid: "S2310656_wal_mc16200_mc16203", sampleType: "WES", }, - S2310659_wal_mc16200_mc16203: { - createdDate: "2018-03-30T11:50:40.079Z", - elasticsearchIndex: null, - datasetName: null, - datasetType: "SNV_INDEL", - individualGuid: "I021476_na19678", - loadedDate: "2018-03-13T13:25:21.551Z", - projectGuid: PROJECT_GUID, - familyGuid: FAMILY_GUID, - sampleGuid: "S2310656_wal_mc16200_mc16203", - sampleId: "WAL_MC16200_MC16203", - isActive: true, - sampleType: "RNA", - }, }, analysisGroupsByGuid: { [ANALYSIS_GROUP_GUID]: { diff --git a/ui/redux/rootReducer.js b/ui/redux/rootReducer.js index 99bf5f2136..ccb080601b 100644 --- a/ui/redux/rootReducer.js +++ b/ui/redux/rootReducer.js @@ -204,12 +204,12 @@ const navigateSavedHashedSearchHelper = (resultsPath, hashKey) => (search, navig export const navigateSavedHashedSearch = navigateSavedHashedSearchHelper('results', 'searchesByHash') export const navigateFamiliesSearch = navigateSavedHashedSearchHelper('families', 'searchFamiliesByHash') -const updateSavedVariant = (values, action = 'create') => (dispatch, getState) => new HttpRequestHelper( +const updateSavedVariant = (values, action = 'create') => dispatch => new HttpRequestHelper( `/api/saved_variant/${action}`, (responseJson) => { dispatch({ type: RECEIVE_DATA, updatesById: responseJson }) }, -).post({ searchHash: getState().currentSearchHash, ...values }) +).post(values) export const updateVariantNote = (values) => { if (values.variantGuids) { @@ -262,7 +262,7 @@ const rootReducer = combineReducers({ familyNotesByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'familyNotesByGuid'), familyDetailsLoading: createSingleObjectReducer(REQUEST_FAMILY_DETAILS), individualsByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'individualsByGuid'), - samplesByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'samplesByGuid'), + datasetsByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'datasetsByGuid'), igvSamplesByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'igvSamplesByGuid'), analysisGroupsByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'analysisGroupsByGuid'), analysisGroupsLoading: loadingReducer(REQUEST_ANALYSIS_GROUPS, RECEIVE_ANALYSIS_GROUPS), diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index 97ef046a96..e9b293b467 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -11,7 +11,7 @@ export const getFamiliesByGuid = state => state.familiesByGuid export const getFamilyNotesByGuid = state => state.familyNotesByGuid export const getFamilyDetailsLoading = state => state.familyDetailsLoading export const getIndividualsByGuid = state => state.individualsByGuid -const getSamplesByGuid = state => state.samplesByGuid +const getDatasetsByGuid = state => state.datasetsByGuid export const getIgvSamplesByGuid = state => state.igvSamplesByGuid export const getAnalysisGroupsByGuid = state => state.analysisGroupsByGuid export const getAnalysisGroupIsLoading = state => state.analysisGroupsLoading.isLoading @@ -58,6 +58,7 @@ const groupEntitiesByProjectGuid = entities => Object.entries(entities).reduce(( }, {}) export const getFamiliesGroupedByProjectGuid = createSelector(getFamiliesByGuid, groupEntitiesByProjectGuid) export const getAnalysisGroupsGroupedByProjectGuid = createSelector(getAnalysisGroupsByGuid, groupEntitiesByProjectGuid) +export const getDatasetsGroupedByProjectGuid = createSelector(getDatasetsByGuid, groupEntitiesByProjectGuid) const groupByFamilyGuid = objs => objs.reduce((acc, o) => { if (!acc[o.familyGuid]) { @@ -69,14 +70,14 @@ const groupByFamilyGuid = objs => objs.reduce((acc, o) => { export const getProjectDatasetTypes = createSelector( getProjectsByGuid, - getSamplesByGuid, - (projectsByGuid, samplesByGuid) => { + getDatasetsByGuid, + (projectsByGuid, datasetsByGuid) => { const projectDatasetTypes = Object.values(projectsByGuid).reduce( (acc, { projectGuid, datasetTypes }) => ({ ...acc, [projectGuid]: datasetTypes }), {}, ) - const sampleDatasetTypes = Object.values(samplesByGuid).reduce( - (acc, { projectGuid, datasetType, isActive }) => { - if (projectDatasetTypes[projectGuid] || !isActive) { + const sampleDatasetTypes = Object.values(datasetsByGuid).reduce( + (acc, { projectGuid, datasetType, activeIndividuals }) => { + if (projectDatasetTypes[projectGuid] || !(activeIndividuals || []).length) { return acc } if (!acc[projectGuid]) { @@ -91,15 +92,25 @@ export const getProjectDatasetTypes = createSelector( ) export const getDatasetsByIndividual = createSelector( - getSamplesByGuid, - samplesByGuid => Object.values(samplesByGuid).sort( + getDatasetsByGuid, + datasetsByGuid => Object.values(datasetsByGuid).sort( (a, b) => a.loadedDate.localeCompare(b.loadedDate), - ).reduce((acc, sample) => { - const { individualGuid, isActive, sampleType, datasetType, loadedDate } = sample - if (!acc[individualGuid]) { - acc[individualGuid] = [] - } - acc[individualGuid].push({ isActive, sampleType, datasetType, loadedDate: loadedDate.split('T')[0] }) + ).reduce((acc, { activeIndividuals, inactiveIndividuals, sampleType, datasetType, loadedDate }) => { + const parsedLoadedDate = loadedDate.split('T')[0] + const activeIndivs = activeIndividuals || [] + activeIndivs.forEach((individualGuid) => { + if (!acc[individualGuid]) { + acc[individualGuid] = [] + } + acc[individualGuid].push({ isActive: true, sampleType, datasetType, loadedDate: parsedLoadedDate }) + }) + const inactiveIndivs = inactiveIndividuals || [] + inactiveIndivs.forEach((individualGuid) => { + if (!acc[individualGuid]) { + acc[individualGuid] = [] + } + acc[individualGuid].push({ isActive: false, sampleType, datasetType, loadedDate: parsedLoadedDate }) + }) return acc }, {}), ) diff --git a/ui/shared/components/panel/MatchmakerPanel.test.js b/ui/shared/components/panel/MatchmakerPanel.test.js index bdba5540d4..8a30161006 100644 --- a/ui/shared/components/panel/MatchmakerPanel.test.js +++ b/ui/shared/components/panel/MatchmakerPanel.test.js @@ -12,7 +12,7 @@ test('shallow-render SubmissionGeneVariants without crashing', () => { const store = configureStore()(STATE1) shallow() }) diff --git a/ui/shared/components/panel/fixtures.js b/ui/shared/components/panel/fixtures.js index 4bca6989a7..f05658a765 100644 --- a/ui/shared/components/panel/fixtures.js +++ b/ui/shared/components/panel/fixtures.js @@ -374,7 +374,7 @@ export const STATE1 = { }, }, analysisGroupsByGuid: {}, - samplesByGuid: {}, + datasetsByGuid: {}, igvSamplesByGuid: { S2310656_wal_mc16200_mc16203: { projectGuid: 'R0237_1000_genomes_demo', diff --git a/ui/shared/components/panel/variants/FamilyVariantTags.jsx b/ui/shared/components/panel/variants/FamilyVariantTags.jsx index 2d53cc6f41..16b888e6ec 100644 --- a/ui/shared/components/panel/variants/FamilyVariantTags.jsx +++ b/ui/shared/components/panel/variants/FamilyVariantTags.jsx @@ -108,7 +108,14 @@ export const taggedByPopup = (tag, title) => (trigger, hideMetadata) => ( ) : {tag.metadata}}
)} - {tag.searchHash &&
Re-run search
} + {tag.searchHash && ( +
+ + Re-run   + {tag.searchName ? `search: "${tag.searchName}"` : 'custom search'} + +
+ )} } /> @@ -393,10 +400,14 @@ const mapStateToProps = (state, ownProps) => { const mapDispatchToProps = (dispatch, ownProps) => ({ dispatchUpdateVariantNote: updates => dispatch( - updateVariantNote({ ...updates, variant: ownProps.variant, familyGuid: ownProps.familyGuid }), + updateVariantNote({ + ...updates, variant: ownProps.variant, searchHash: ownProps.searchHash, familyGuid: ownProps.familyGuid, + }), ), dispatchUpdateFamilyVariantTags: updates => dispatch( - updateVariantTags({ ...updates, variant: ownProps.variant, familyGuid: ownProps.familyGuid }), + updateVariantTags({ + ...updates, variant: ownProps.variant, searchHash: ownProps.searchHash, familyGuid: ownProps.familyGuid, + }), ), dispatchUpdateFamilyVariantFunctionalTags: updates => dispatch( updateVariantTags({ ...updates, variant: ownProps.variant, familyGuid: ownProps.familyGuid }, 'functional_data'), diff --git a/ui/shared/components/panel/variants/RnaSeqTpm.jsx b/ui/shared/components/panel/variants/RnaSeqTpm.jsx index fe1612a680..6185d61d3a 100644 --- a/ui/shared/components/panel/variants/RnaSeqTpm.jsx +++ b/ui/shared/components/panel/variants/RnaSeqTpm.jsx @@ -4,17 +4,17 @@ import { extent, max, median, min, quantile } from 'd3-array' import { scaleBand, scaleLinear } from 'd3-scale' import { HttpRequestHelper } from 'shared/utils/httpRequestHelper' -import { TISSUE_DISPLAY } from 'shared/utils/constants' +import { TISSUE_DISPLAY, PRODUCT_DISPLAY } from 'shared/utils/constants' import { initializeD3, Tooltip } from 'shared/components/graph/d3Utils' import GtexLauncher from 'shared/components/graph/GtexLauncher' const BOX_WIDTH = 100 -const PLOT_WIDTH = 550 +const PLOT_WIDTH = 500 const PLOT_HEIGHT = 350 const MARGINS = { left: 40, top: 0, - bottom: 100, + bottom: 130, } // Code adapted from https://github.com/broadinstitute/gtex-viz/blob/8d65862fbe7e5ab9b4d5be419568754e0d17bb07/src/modules/Boxplot.js @@ -56,7 +56,7 @@ const renderBoxplot = (allData, containerElement) => { const svg = initializeD3( containerElement, { width: PLOT_WIDTH, height: PLOT_HEIGHT }, - { ...MARGINS, right: bandwidth / 2 + 10 }, + { ...MARGINS, right: bandwidth / 2 + 60 }, scales, { x: { offset: bandwidth / 2 }, y: { text: 'TPM' } }, ) @@ -189,17 +189,16 @@ const renderGtex = (gtexExpressionData, familyExpressionData, containerElement) const gtexByTissue = ((gtexExpressionData || {}).data || []).reduce((acc, { data, tissueSiteDetailId }) => ({ ...acc, [GTEX_TISSUE_LOOKUP[tissueSiteDetailId]]: data, }), {}) - const boxplotData = Object.entries(familyExpressionData).reduce(( - acc, [tissue, { rdgData, myData, individualData }], - ) => ([ - ...acc, - ...[ - { data: gtexByTissue[tissue], label: 'GTEx' }, - { data: rdgData, label: 'Broad RDG' }, - { data: myData, label: 'My Data' }, - ...Object.entries(individualData).map(([individual, tpm]) => ({ data: [tpm], label: individual, medianColor: '#000080' })), - ].filter(({ data }) => data).map(({ label, ...d }) => ({ label: `${label} - ${TISSUE_DISPLAY[tissue]}`, ...d })), - ]), []) + const boxplotData = Object.entries(familyExpressionData).reduce((acc, [tissue, tissueData]) => ([ + ...acc, ...[ + { data: gtexByTissue[tissue], label: `GTEx - ${TISSUE_DISPLAY[tissue]}` }, + ...Object.entries(tissueData).reduce((tAcc, [sequencingProduct, { rdgData, myData, individualData }]) => ([ + ...tAcc, + { data: rdgData, label: 'Broad RDG' }, + { data: myData, label: 'My Data' }, + ...Object.entries(individualData).map(([individual, tpm]) => ({ data: [tpm], label: individual, medianColor: '#000080' })), + ].filter(({ data }) => data).map(({ label, ...d }) => ({ label: `${label} - ${TISSUE_DISPLAY[tissue]} - ${PRODUCT_DISPLAY[sequencingProduct]}`, ...d }))), []), + ]]), []) renderBoxplot(boxplotData, containerElement) } diff --git a/ui/shared/components/panel/variants/Variants.jsx b/ui/shared/components/panel/variants/Variants.jsx index 7bad475092..bd16675dfa 100644 --- a/ui/shared/components/panel/variants/Variants.jsx +++ b/ui/shared/components/panel/variants/Variants.jsx @@ -78,7 +78,7 @@ const tagFamily = tag => ( const VariantLayout = ( { variant, compoundHetToggle, mainGeneId, isCompoundHet, linkToSavedVariants, topContent, - bottomContent, children, ...rowProps + bottomContent, children, searchHash, ...rowProps }, ) => { const coreVariant = Array.isArray(variant) ? variant[0] : variant @@ -95,6 +95,7 @@ const VariantLayout = ( variant={variant} isCompoundHet={isCompoundHet} linkToSavedVariants={linkToSavedVariants} + searchHash={searchHash} /> @@ -131,6 +132,7 @@ VariantLayout.propTypes = { topContent: PropTypes.node, bottomContent: PropTypes.node, children: PropTypes.node, + searchHash: PropTypes.string, } export const Variant = React.memo(( diff --git a/ui/shared/components/panel/view-pedigree-image/LazyPedigreeImagePanel.jsx b/ui/shared/components/panel/view-pedigree-image/LazyPedigreeImagePanel.jsx index 5662693c92..3e445935a3 100644 --- a/ui/shared/components/panel/view-pedigree-image/LazyPedigreeImagePanel.jsx +++ b/ui/shared/components/panel/view-pedigree-image/LazyPedigreeImagePanel.jsx @@ -201,26 +201,24 @@ class BasePedigreeImage extends React.PureComponent { dataset.filter( ({ mother, father }) => mother || father, ).forEach((indiv) => { - if (indiv.mother && indiv.father) { - // pedigree js incorrectly considers people whose parents are different generations consanguineous - const mother = nameMap[indiv.mother] - const father = nameMap[indiv.father] - let missingGenParent = null - if (mother.top_level && !father.top_level) { - missingGenParent = mother - } else if (father.top_level && !mother.top_level) { - missingGenParent = father - } - if (missingGenParent) { - missingGenParent.top_level = false - this.addParent(`${missingGenParent.name}-father-placeholder`, missingGenParent, newParents, true) - this.addParent(`${missingGenParent.name}-mother-placeholder`, missingGenParent, newParents, false) - } - return + if (!indiv.mother || !indiv.father) { + const placeholderId = `${indiv.mother || indiv.father}-spouse-placeholder` + this.addParent(placeholderId, indiv, newParents, !indiv.father) + } + // pedigree js incorrectly considers people whose parents are different generations consanguineous + const mother = nameMap[indiv.mother] || newParents[indiv.mother] + const father = nameMap[indiv.father] || newParents[indiv.father] + let missingGenParent = null + if (mother.top_level && !father.top_level) { + missingGenParent = mother + } else if (father.top_level && !mother.top_level) { + missingGenParent = father + } + if (missingGenParent) { + missingGenParent.top_level = false + this.addParent(`${missingGenParent.name}-father-placeholder`, missingGenParent, newParents, true) + this.addParent(`${missingGenParent.name}-mother-placeholder`, missingGenParent, newParents, false) } - - const placeholderId = `${indiv.mother || indiv.father}-spouse-placeholder` - this.addParent(placeholderId, indiv, newParents, !indiv.father) }) return this.yobToAge([...dataset, ...Object.values(newParents)]) diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index 40a82f0f53..908acc8294 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -2119,6 +2119,12 @@ export const TISSUE_DISPLAY = { B: 'Brain', } +export const PRODUCT_DISPLAY = { + K: 'Kinnex', + T: 'TruSeq', + W: 'Watchmaker', +} + export const RNASEQ_JUNCTION_PADDING = 200 export const FAQ_PATH = '/faq' diff --git a/vlm/test_vlm.py b/vlm/test_vlm.py index 7889cddbeb..9b0a1f02e3 100644 --- a/vlm/test_vlm.py +++ b/vlm/test_vlm.py @@ -32,7 +32,7 @@ def inject_fixtures(self, caplog): async def test_match(self, mocked_responses): mocked_responses.post( - 'https://vlm-auth.us.auth0.com/oauth/token', payload={'access_token': 'test_token'}, repeat=True, + 'https://vlm-auth.us.auth0.com/oauth/token', payload={'access_token': 'test_token'}, repeat=True, # nosec ) mocked_responses.get( f'https://vlm-auth.us.auth0.com/api/v2/clients/{REQUESTER_CLIENT_ID}', @@ -106,7 +106,7 @@ async def test_match(self, mocked_responses): mocked_responses.assert_called_with( 'https://vlm-auth.us.auth0.com/oauth/token', method='POST', - json={'client_id': 'unknown_client_id', 'client_secret': 'unknown_client_secret', 'audience': 'https://vlm-auth.us.auth0.com/api/v2/', 'grant_type': 'client_credentials'}, + json={'client_id': 'unknown_client_id', 'client_secret': 'unknown_client_secret', 'audience': 'https://vlm-auth.us.auth0.com/api/v2/', 'grant_type': 'client_credentials'}, # nosec ) mocked_responses.assert_called_with( f'https://vlm-auth.us.auth0.com/api/v2/clients/{REQUESTER_CLIENT_ID}', @@ -292,7 +292,7 @@ async def test_match_error(self, mocked_responses): self.assertEqual(resp.reason, 'Credential Check Error') self.assertEqual(self._caplog.messages[-3], "Credential Check Error: 400 - {'error': 'invalid_request'}") - mocked_responses.post('https://vlm-auth.us.auth0.com/oauth/token', payload={'access_token': 'test_token'}, repeat=True) + mocked_responses.post('https://vlm-auth.us.auth0.com/oauth/token', payload={'access_token': 'test_token'}, repeat=True) # nosec async with self.client.request('GET', '/vlm/match', headers=headers) as resp: self.assertEqual(resp.status, 403) self.assertEqual(resp.reason, 'Invalid Client ID abc123') diff --git a/wsgi.py b/wsgi.py index 1fd7e20bda..a8f0b255bc 100644 --- a/wsgi.py +++ b/wsgi.py @@ -1,18 +1,3 @@ -""" -WSGI config for xbrowse_server project. - -This module contains the WSGI application used by Django's development server -and any production WSGI deployments. It should expose a module-level variable -named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover -this application via the ``WSGI_APPLICATION`` setting. - -Usually you will have the standard Django WSGI application here, but it also -might make sense to replace the whole Django WSGI application with a custom one -that later delegates to the Django one. For example, you could introduce WSGI -middleware here, or combine a Django application with an application of another -framework. - -""" import os os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")