diff --git a/clickhouse_search/all_search_tests.py b/clickhouse_search/all_search_tests.py index 74d4c11442..f367ba0f60 100644 --- a/clickhouse_search/all_search_tests.py +++ b/clickhouse_search/all_search_tests.py @@ -2408,7 +2408,7 @@ def test_gene_variant_lookup(self): cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}], annotations=annotations, freqs=freqs, locus=locus, project_families=[], export_data=[ EXPORT_DATA[0][:27], EXPORT_DATA[4][:24] + ['3 Families', '', ''], - ], + ], gene_counts={'ENSG00000097046': {'total': 1, 'families': {}}}, ) freqs = {'callset': freqs['callset']} @@ -2443,6 +2443,10 @@ def test_gene_variant_lookup(self): {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}, ], annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', project_families=[], + gene_counts={ + 'ENSG00000097046': {'total': 2, 'families': {}}, + 'ENSG00000177000': {'total': 1, 'families': {}}, + }, ) self.login_collaborator() @@ -2455,7 +2459,10 @@ def test_gene_variant_lookup(self): {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][3]}, {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}, ], - annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', + annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', gene_counts={ + 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, + 'ENSG00000177000': {'total': 1, 'families': {'F000003_3': 1}}, + }, ) locus['rawItems'] = 'ENSG00000171621' diff --git a/clickhouse_search/fixtures/variant_details_for_update.json b/clickhouse_search/fixtures/variant_details_for_update.json index d8249f0296..45fe5e5d9f 100644 --- a/clickhouse_search/fixtures/variant_details_for_update.json +++ b/clickhouse_search/fixtures/variant_details_for_update.json @@ -83,7 +83,7 @@ "key": 6, "variant_id": "1-94818-T-C", "rsid": null, - "caid": null, + "caid": "CA16717153", "lifted_over_chrom": "1", "lifted_over_pos": 94818, "sorted_regulatory_feature_consequences": [], diff --git a/clickhouse_search/management/commands/register_caids.py b/clickhouse_search/management/commands/register_caids.py index 72d13929e6..e0cd9a8c34 100644 --- a/clickhouse_search/management/commands/register_caids.py +++ b/clickhouse_search/management/commands/register_caids.py @@ -224,7 +224,7 @@ def handle_api_response( def register_caids( genome_version: Literal[GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38], variants: Union[list[VariantDetailsGRCh37SnvIndel], list[VariantDetailsSnvIndel]], -) -> int: +) -> tuple[int, Union[list[VariantDetailsGRCh37SnvIndel], list[VariantDetailsSnvIndel]]]: rows = list(ALLELE_REGISTRY_HEADERS[genome_version]) # NB: new copy of the list for variant in variants: chrom, pos, ref, alt = variant.variant_id.split("-") @@ -254,10 +254,14 @@ def register_caids( ) mapped_variants = handle_api_response(genome_version, res) max_key_id = -1 + update_variants = [] for variant in variants: - variant.caid = mapped_variants.get(variant.variant_id, None) + caid = mapped_variants.get(variant.variant_id) + if caid: + variant.caid = caid + update_variants.append(variant) max_key_id = max(max_key_id, variant.key_id) - return max_key_id + return max_key_id, update_variants def join_series(model: Union[VariantDetailsSnvIndel, VariantDetailsGRCh37SnvIndel], min_: int, max_: int): table = model._meta.db_table @@ -316,8 +320,8 @@ def handle(self, *args, **options): break try: - max_key = register_caids(genome_version, variants) - variant_details_model.objects.using('clickhouse_write').bulk_update(variants, ["caid"]) + max_key, update_variants = register_caids(genome_version, variants) + variant_details_model.objects.using('clickhouse_write').bulk_update(update_variants, ["caid"]) # Save current key on every iteration curr_key = max_key version_obj.version = curr_key diff --git a/clickhouse_search/management/tests/register_caids_tests.py b/clickhouse_search/management/tests/register_caids_tests.py index d0c27f257f..927d72e3ee 100644 --- a/clickhouse_search/management/tests/register_caids_tests.py +++ b/clickhouse_search/management/tests/register_caids_tests.py @@ -269,8 +269,16 @@ def test_register_caids(self, mock_safe_post_to_slack, mock_logger): 'MESSAGE: 1\t10469\trs370233998\tC\tG\t.\t.\t.\n' 'INPUT_LINE: Cannot align NC_000001.10 [10468,10469).', ) - vd = VariantDetailsSnvIndel.objects.get(variant_id='1-91511686-T-G') - self.assertEqual(vd.caid, 'CA997563840') + self.assertListEqual(list(VariantDetailsSnvIndel.objects.order_by('key').values_list('variant_id', 'caid')), [ + ('1-10439-AC-A', 'CA16717152'), + ('1-38724419-T-G', None), + ('1-91502721-G-A', None), + ('1-91511686-T-G', 'CA997563840'), + ('1-10146-ACC-A', 'CA997563845'), + ('1-94818-T-C', 'CA16717153'), + ('7-143270172-A-G', None), + ('7-9310123-T-C', None) + ]) # Ensure re-calling is a no-op responses.reset() diff --git a/matchmaker/views/matchmaker_api.py b/matchmaker/views/matchmaker_api.py index be0ea94e25..540f20f455 100644 --- a/matchmaker/views/matchmaker_api.py +++ b/matchmaker/views/matchmaker_api.py @@ -20,8 +20,8 @@ from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.orm_to_json_utils import _get_json_for_model, get_json_for_saved_variants, \ get_json_for_matchmaker_submission, get_json_for_matchmaker_submissions -from seqr.views.utils.permissions_utils import check_mme_permissions, check_project_permissions, analyst_required, \ - has_project_permissions, login_and_policies_required, get_project_and_check_permissions +from seqr.views.utils.permissions_utils import check_mme_permissions, check_family_view_permission, analyst_required, \ + has_family_view_permission, login_and_policies_required, get_project_and_check_edit_permission from settings import BASE_URL, MME_ACCEPT_HEADER, MME_NODES, MME_DEFAULT_CONTACT_EMAIL, \ MME_SLACK_SEQR_MATCH_NOTIFICATION_CHANNEL, MME_SLACK_ALERT_NOTIFICATION_CHANNEL, VLM_SEND_EMAIL @@ -36,7 +36,7 @@ @login_and_policies_required def get_individual_mme_matches(request, submission_guid): submission = MatchmakerSubmission.objects.get(guid=submission_guid) - project = check_mme_permissions(submission, request.user) + genome_version = check_mme_permissions(submission, request.user) results = MatchmakerResult.objects.filter(submission=submission) @@ -51,7 +51,7 @@ def get_individual_mme_matches(request, submission_guid): variants = get_json_for_saved_variants( SavedVariant.objects.filter(guid__in=variant_guids), additional_values={ - 'genomeVersion': Value(project.genome_version), + 'genomeVersion': Value(genome_version), 'selectedMainTranscript': F('main_transcript'), 'xposEnd': F('xpos_end'), }, @@ -273,7 +273,7 @@ def update_mme_submission(request, submission_guid=None): if not individual_guid: return create_json_response({}, status=400, reason='Individual is required for a new submission') individual = Individual.objects.get(guid=individual_guid) - check_project_permissions(individual.family.project, request.user) + check_family_view_permission(individual.family, request.user) submission = create_model_from_json(MatchmakerSubmission, { 'individual': individual, 'submission_id': individual.guid, @@ -390,7 +390,7 @@ def send_mme_contact_email(request, matchmaker_result_guid): @login_and_policies_required def update_mme_project_contact(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) + project = get_project_and_check_edit_permission(project_guid, request.user) request_json = json.loads(request.body) contact = (request_json.get('contact') or '').strip() @@ -434,7 +434,7 @@ def _parse_mme_results(submission, saved_results, user, additional_genes=None, r result['matchStatus'] = _get_json_for_model(result_model) if result_model.originating_submission: originating_family = result_model.originating_submission.individual.family - if has_project_permissions(originating_family.project, user): + if has_family_view_permission(originating_family, user): result['originatingSubmission'] = { 'originatingSubmissionGuid': result_model.originating_submission.guid, 'familyGuid': originating_family.guid, diff --git a/seqr/urls.py b/seqr/urls.py index 5c6bf5cef7..0f4acf20e9 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -125,6 +125,7 @@ family_metadata, \ variant_metadata, \ gregor_export, \ + sample_stats_download, \ seqr_stats from seqr.views.apis.summary_data_api import success_story, saved_variants_page, mme_details, hpo_summary_data, \ bulk_update_family_external_analysis, individual_metadata @@ -325,6 +326,7 @@ 'report/variant_metadata/(?P[^/]+)': variant_metadata, 'report/gregor': gregor_export, 'report/seqr_stats': seqr_stats, + 'report/sample_stats_download': sample_stats_download, 'data_management/get_all_users': get_all_users, 'data_management/update_rna_seq': update_rna_seq, diff --git a/seqr/utils/add_data_utils.py b/seqr/utils/add_data_utils.py index 2feefa04fa..81dc2e50a7 100644 --- a/seqr/utils/add_data_utils.py +++ b/seqr/utils/add_data_utils.py @@ -90,7 +90,7 @@ def trigger_rebuild_gt_stats(project, user): def trigger_data_loading(projects: list[Project], individual_ids: list[int], sample_type: str, dataset_type: str, genome_version: str, data_path: str, user: User, raise_error: bool = False, skip_expect_tdr_metrics: bool = True, - skip_check_sex_and_relatedness: bool = True, vcf_sample_id_map=None, + skip_check_sex_and_relatedness: bool = True, validations_to_skip: list[str] = None, vcf_sample_id_map=None, success_message: str = None, error_message: str = None, success_slack_channel: str = SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL): variables = { 'projects_to_run': sorted([p.guid for p in projects]) if projects else None, @@ -99,11 +99,12 @@ def trigger_data_loading(projects: list[Project], individual_ids: list[int], sam 'callset_path': data_path, 'sample_type': sample_type, } - bool_variables = { + conditional_variables = { 'skip_check_sex_and_relatedness': skip_check_sex_and_relatedness, 'skip_expect_tdr_metrics': skip_expect_tdr_metrics, + 'validations_to_skip': _valid_validation_to_skip(validations_to_skip, user), } - variables.update({k: v for k, v in bool_variables.items() if v}) + variables.update({k: v for k, v in conditional_variables.items() if v}) file_path = _get_pedigree_path(genome_version, sample_type, dataset_type) _upload_data_loading_files(individual_ids, vcf_sample_id_map or {}, user, file_path, raise_error) _write_gene_id_file(user) @@ -126,6 +127,16 @@ def trigger_data_loading(projects: list[Project], individual_ids: list[int], sam return success +def _valid_validation_to_skip(validations_to_skip: list[str], user: User) -> list[str]: + invalid_validations = [validation for validation in (validations_to_skip or []) if validation not in { + 'validate_expected_contig_frequency', 'validate_sample_type', 'validate_no_duplicate_variants', + }] + if invalid_validations: + validations_to_skip = [validation for validation in validations_to_skip if validation not in invalid_validations] + logger.error(f'Omitted invalid skip validations from loading request: {", ".join(invalid_validations)}', user) + return validations_to_skip + + def _enqueue_pipeline_request(name: str, variables: dict, user: User, raise_error: bool = True, log_error: bool = True): response = None error = None @@ -185,7 +196,7 @@ def _upload_data_loading_files(individual_ids: list[int], vcf_sample_id_map: dic def _write_gene_id_file(user): file_name = 'db_id_to_gene_id' - if does_file_exist(f'{LOADING_DATASETS_DIR}/{file_name}.csv.gz'): + if does_file_exist(f'{LOADING_DATASETS_DIR}/{file_name}.csv.gz', user): return gene_data_loaded = (GeneInfo.objects.filter(gencode_release=int(GeneInfo.CURRENT_VERSION)).exists() and diff --git a/seqr/views/apis/analysis_group_api.py b/seqr/views/apis/analysis_group_api.py index 90322da56c..dd88951653 100644 --- a/seqr/views/apis/analysis_group_api.py +++ b/seqr/views/apis/analysis_group_api.py @@ -4,7 +4,7 @@ from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json from seqr.views.utils.orm_to_json_utils import get_json_for_analysis_group -from seqr.views.utils.permissions_utils import get_project_and_check_permissions, login_and_policies_required +from seqr.views.utils.permissions_utils import get_project_and_check_edit_permission, login_and_policies_required REQUIRED_FIELDS = {'name': 'Name', 'familyGuids': 'Families'} @@ -12,7 +12,7 @@ def _update_analysis_group(request, project_guid, analysis_group_guid, model_cls, required_fields, is_dynamic=False, validate_body=lambda x: None, post_process_model=lambda x: None): - project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) + project = get_project_and_check_edit_permission(project_guid, request.user) request_json = json.loads(request.body) missing_fields = [field for field in required_fields.keys() if not request_json.get(field)] @@ -79,7 +79,7 @@ def update_dynamic_analysis_group_handler(request, project_guid, analysis_group_ @login_and_policies_required def delete_analysis_group_handler(request, project_guid, analysis_group_guid, model_cls=AnalysisGroup): - project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) + project = get_project_and_check_edit_permission(project_guid, request.user) model_cls.objects.get(guid=analysis_group_guid, project=project).delete_model(request.user, user_can_delete=True) return create_json_response({'analysisGroupsByGuid': {analysis_group_guid: None}}) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 2aa72bd842..915250cf34 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -304,19 +304,18 @@ def load_data(request): if errors: raise ErrorsWarningsException(errors) - is_local = True - success_message = None - error_message = None - if AirtableSession.is_airtable_enabled(): - is_local = False - success_message = f'*{request.user.email}* triggered loading internal {sample_type} {dataset_type} data for {len(individual_ids)} samples in {len(projects)} projects ({"; ".join(sorted(project_counts))})' - error_message = f'ERROR triggering internal {sample_type} {dataset_type} loading' + loading_kwargs = { + 'skip_check_sex_and_relatedness': request_json.get('skipSRChecks', False), + 'skip_expect_tdr_metrics': request_json.get('skipTDR', False), + 'vcf_sample_id_map': vcf_sample_id_map, + 'success_message': f'*{request.user.email}* triggered loading internal {sample_type} {dataset_type} data for {len(individual_ids)} samples in {len(projects)} projects ({"; ".join(sorted(project_counts))})', + 'error_message': f'ERROR triggering internal {sample_type} {dataset_type} loading', + } if AirtableSession.is_airtable_enabled() else {'raise_error': True} success = trigger_data_loading( projects_by_guid.values(), individual_ids, sample_type, dataset_type, request_json['genomeVersion'], - _callset_path(request_json), user=request.user, - skip_check_sex_and_relatedness=request_json.get('skipSRChecks', False), vcf_sample_id_map=vcf_sample_id_map, - raise_error=is_local, skip_expect_tdr_metrics=is_local, success_message=success_message, error_message=error_message, + _callset_path(request_json), user=request.user, validations_to_skip=request_json.get('validationsToSkip'), + **loading_kwargs, ) return create_json_response({'success': success}) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index e5b397a934..783029bd83 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -161,6 +161,7 @@ 'filePath': '/callset.vcf', 'sampleType': 'WES', 'genomeVersion': '38', + 'validationsToSkip': ['validate_sample_type', 'validate_no_duplicate_variants', 'validate_something_else'] } AIRTABLE_SAMPLE_RECORDS = { @@ -1117,7 +1118,7 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_gzip_open, mock_mkdir, m mock_temp_dir.return_value.__enter__.return_value = '/mock/tmp' body = {**self.REQUEST_BODY, 'projects': [ json.dumps(option) for option in self.PROJECT_OPTIONS + [{'projectGuid': 'R0005_not_project'}] - ], 'vcfSamples': self.VCF_SAMPLES, 'skipSRChecks': True} + ], 'vcfSamples': self.VCF_SAMPLES} response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': 'The following projects are invalid: R0005_not_project'}) @@ -1133,8 +1134,8 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_gzip_open, mock_mkdir, m ) self.assertDictEqual(response.json(), {'success': True}) - self._assert_expected_load_data_requests(sample_type='WES', skip_check_sex_and_relatedness=True) - self._has_expected_ped_files(mock_open, mock_gzip_open, mock_mkdir, 'SNV_INDEL', sample_type='WES', has_remap=bool(self.MOCK_AIRTABLE_KEY)) + load_data_logs = self._assert_expected_load_data_requests(sample_type='WES', skip_check_sex_and_relatedness=True, validations_to_skip=['validate_sample_type', 'validate_no_duplicate_variants']) + ped_file_logs = self._has_expected_ped_files(mock_open, mock_gzip_open, mock_mkdir, 'SNV_INDEL', sample_type='WES', has_remap=bool(self.MOCK_AIRTABLE_KEY)) variables = { 'projects_to_run': [ @@ -1146,11 +1147,21 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_gzip_open, mock_mkdir, m 'callset_path': f'{self.TRIGGER_CALLSET_DIR}/callset.vcf', 'sample_type': 'WES', 'skip_check_sex_and_relatedness': True, + 'validations_to_skip': ['validate_sample_type', 'validate_no_duplicate_variants'], } if self.SKIP_TDR: variables['skip_expect_tdr_metrics'] = True self._assert_success_notification(variables) + self.assert_json_logs(self.data_manager_user if load_data_logs else self.pm_user, [ + (log, None) for log in load_data_logs + ] + [ + ('Omitted invalid skip validations from loading request: validate_something_else', { + 'severity': 'ERROR', + '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', + }), + ] + ped_file_logs + [('Triggered Loading Pipeline', {'detail': variables})]) + # Test loading trigger error self._set_file_not_found(has_mv_commands=True) mock_open.reset_mock() @@ -1159,8 +1170,12 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_gzip_open, mock_mkdir, m responses.calls.reset() self.reset_logs() - del body['skipSRChecks'] + body.pop('skipSRChecks', None) + del body['validationsToSkip'] + body['skipTDR'] = True del variables['skip_check_sex_and_relatedness'] + del variables['validations_to_skip'] + variables['skip_expect_tdr_metrics'] = True body.update({'datasetType': 'SV', 'filePath': f'{self.CALLSET_DIR}/sv_callset.vcf'}) self._trigger_error(url, body, variables, mock_open, mock_gzip_open, mock_mkdir) @@ -1171,6 +1186,7 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_gzip_open, mock_mkdir, m mock_mkdir.reset_mock() body.update({'sampleType': 'WGS', 'projects': [json.dumps(self.PROJECT_OPTION)], 'vcfSamples': VCF_SAMPLES}) del body['datasetType'] + del body['skipTDR'] response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self._test_load_single_project(mock_open, mock_gzip_open, mock_mkdir, response, url=url, body=body) @@ -1203,7 +1219,7 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_gzip_open, mock_mkdir, m }) self.assertEqual(len(responses.calls), 0) - def _assert_expected_load_data_requests(self, dataset_type='SNV_INDEL', sample_type='WGS', trigger_error=False, skip_project=False, skip_check_sex_and_relatedness=False): + def _assert_expected_load_data_requests(self, dataset_type='SNV_INDEL', sample_type='WGS', trigger_error=False, skip_project=False, skip_check_sex_and_relatedness=False, skip_tdr=False, validations_to_skip=None): projects = [PROJECT_GUID, NON_ANALYST_PROJECT_GUID] if skip_project: projects = projects[1:] @@ -1214,16 +1230,19 @@ def _assert_expected_load_data_requests(self, dataset_type='SNV_INDEL', sample_t 'dataset_type': dataset_type, 'reference_genome': 'GRCh38', } - if self.SKIP_TDR: + if validations_to_skip: + body['validations_to_skip'] = validations_to_skip + if skip_tdr or self.SKIP_TDR: body['skip_expect_tdr_metrics'] = True - if skip_check_sex_and_relatedness: + if skip_check_sex_and_relatedness or self.SKIP_TDR: body['skip_check_sex_and_relatedness'] = True self.assertDictEqual(json.loads(responses.calls[-1].request.body), body) + return [] def _trigger_error(self, url, body, variables, mock_open, mock_gzip_open, mock_mkdir): responses.add(responses.POST, PIPELINE_RUNNER_URL, status=400) response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self._assert_expected_load_data_requests(trigger_error=True, dataset_type='GCNV', sample_type='WES') + self._assert_expected_load_data_requests(trigger_error=True, dataset_type='GCNV', sample_type='WES', skip_tdr=True) self._assert_trigger_error(response, body, variables, response_body={ 'error': f'400 Client Error: Bad Request for url: {PIPELINE_RUNNER_URL}' }) @@ -1275,6 +1294,7 @@ def _has_expected_ped_files(self, mock_open, mock_gzip_open, mock_mkdir, dataset ['R0004_non_analyst_project', 'F000014_14', 'fam14', 'NA21234', '', '', 'F'] + (['ABC123'] if has_remap else []), ['R0004_non_analyst_project', 'F000014_14', 'fam14', 'NA21987', '', '', 'M'] + ([''] if has_remap else []), ]) + return [] def _test_load_single_project(self, mock_open, mock_gzip_open, mock_mkdir, response, *args, **kwargs): self.assertEqual(response.status_code, 200) @@ -1388,23 +1408,23 @@ def _assert_expected_get_projects_requests(self): def _assert_expected_load_data_requests(self, *args, **kwargs): self.assertEqual(len(responses.calls), 1) - super()._assert_expected_load_data_requests(*args, **kwargs) + return super()._assert_expected_load_data_requests(*args, **kwargs) @staticmethod def _local_pedigree_path(dataset_type, sample_type): return f'/local_datasets/GRCh38/{dataset_type}/pedigrees/{sample_type}' def _has_expected_ped_files(self, mock_open, mock_gzip_open, mock_mkdir, dataset_type, *args, sample_type='WGS', has_gene_id_file=False, **kwargs): - super()._has_expected_ped_files(mock_open, mock_gzip_open, mock_mkdir, dataset_type, *args, sample_type, has_gene_id_file=has_gene_id_file, **kwargs) + logs = super()._has_expected_ped_files(mock_open, mock_gzip_open, mock_mkdir, dataset_type, *args, sample_type, has_gene_id_file=has_gene_id_file, **kwargs) call_paths = [self._local_pedigree_path(dataset_type, sample_type)] if not has_gene_id_file: call_paths.append(self.LOCAL_WRITE_DIR) self.assertEqual(mock_mkdir.call_count, len(call_paths)) mock_mkdir.assert_has_calls([mock.call(call_path, exist_ok=True) for call_path in call_paths]) + return logs def _assert_success_notification(self, variables): - self.maxDiff = None - self.assert_json_logs(self.pm_user, [('Triggered Loading Pipeline', {'detail': variables})]) + pass def _trigger_error(self, url, body, variables, mock_open, mock_gzip_open, mock_mkdir): super()._trigger_error(url, body, variables, mock_open, mock_gzip_open, mock_mkdir) @@ -1474,6 +1494,7 @@ class AnvilDataManagerAPITest(AnvilAuthenticationTestCase, DataManagerAPITest): **CORE_REQUEST_BODY, 'filePath': CALLSET_DIR + CORE_REQUEST_BODY['filePath'], 'datasetType': 'SNV_INDEL', + 'skipSRChecks': True, } VCF_SAMPLES = [s for s in VCF_SAMPLES if s != 'NA21234'] @@ -1557,6 +1578,7 @@ def _assert_expected_pm_access(self, get_response, *args, **kwargs): self.assert_json_logs(self.pm_user, [ ('PermissionDenied: Error: To access RDG airtable user must login with Broad email.', {'severity': 'WARNING'}) ]) + self.reset_logs() self.login_data_manager_user() return super()._assert_expected_pm_access(get_response, *args, **kwargs) @@ -1564,15 +1586,19 @@ def _assert_expected_load_data_requests(self, *args, dataset_type='SNV_INDEL', s num_calls = 1 is_gcnv = dataset_type == 'GCNV' required_sample_field = 'gCNV_CallsetPath' if is_gcnv else None + logs = [] if not skip_project: self._assert_expected_airtable_call(required_sample_field, 'R0001_1kg') num_calls += 1 + logs += ['Fetching Samples records 0-2 from airtable', 'Fetched 6 Samples records from airtable'] if (not is_gcnv) and (not skip_project): self._assert_expected_airtable_vcf_id_call(required_sample_field, call_index=1) num_calls += 1 + logs += ['Fetching Samples records 0-4 from airtable', 'Fetched 6 Samples records from airtable'] self.assertEqual(len(responses.calls), num_calls) super()._assert_expected_load_data_requests(*args, dataset_type=dataset_type, skip_project=skip_project, **kwargs) + return logs def _assert_expected_airtable_call(self, required_sample_field, project_guid, call_index=0, additional_filter=None, additional_pdo_statuses='', additional_fields=None): airtable_filters = [ @@ -1697,19 +1723,22 @@ def _has_expected_ped_files(self, mock_open, mock_gzip_open, mock_mkdir, dataset super()._has_expected_ped_files(mock_open, mock_gzip_open, mock_mkdir, dataset_type, sample_type, has_gene_id_file=has_gene_id_file, **kwargs) mock_mkdir.assert_not_called() - expected_calls = [mock.call( + expected_calls = [ f'gsutil mv /mock/tmp/* gs://seqr-loading-temp/v3.1/GRCh38/{dataset_type}/pedigrees/{sample_type}/', - stdout=-1, stderr=-2, shell=True, # nosec - ), mock.call( - 'gsutil ls gs://seqr-loading-temp/v3.1/db_id_to_gene_id.csv.gz', stdout=-1, stderr=-2, shell=True, # nosec - )] + 'gsutil ls gs://seqr-loading-temp/v3.1/db_id_to_gene_id.csv.gz', + ] if not has_gene_id_file: - expected_calls.append(mock.call( - 'gsutil mv /mock/tmp/* gs://seqr-loading-temp/v3.1/', stdout=-1, stderr=-2, shell=True, # nosec - )) + expected_calls.append( + 'gsutil mv /mock/tmp/* gs://seqr-loading-temp/v3.1/', + ) self.assertEqual(self.mock_subprocess.call_count, len(expected_calls)) - self.mock_subprocess.assert_has_calls(expected_calls) + self.mock_subprocess.assert_has_calls([ + mock.call(call, stdout=-1, stderr=-2, shell=True) for call in expected_calls # nosec + ]) self.mock_subprocess.reset_mock() + logs = [(f'==> {call}', None) for call in expected_calls] + logs.insert(2, ('CommandException: One or more URLs matched no objects', {'severity': 'WARNING'})) + return logs def _assert_write_pedigree_error(self, response): self.assertEqual(response.status_code, 200) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 493bc8a380..e0f079aa0c 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -20,10 +20,10 @@ from seqr.views.utils.project_context_utils import add_families_context, families_discovery_tags, add_project_tag_types, \ MME_TAG_NAME from seqr.models import Family, FamilyAnalysedBy, Individual, FamilyNote, Dataset, VariantTag, AnalysisGroup, RnaSeqTpm, \ - PhenotypePrioritization, Project, RnaSample -from seqr.views.utils.permissions_utils import check_project_permissions, get_project_and_check_pm_permissions, \ + PhenotypePrioritization, RnaSample +from seqr.views.utils.permissions_utils import check_project_edit_permission, get_project_and_check_pm_permissions, \ login_and_policies_required, user_is_analyst, has_case_review_permissions, external_anvil_project_can_edit, \ - get_internal_projects, get_project_guids_user_can_view + get_internal_projects, get_project_guids_user_can_view, check_family_view_permission from seqr.views.utils.terra_api_utils import anvil_enabled from seqr.views.utils.variant_utils import get_phenotype_prioritization, get_omim_intervals_query, DISCOVERY_CATEGORY from seqr.utils.xpos_utils import get_chrom_pos @@ -39,7 +39,7 @@ def family_page_data(request, family_guid): families = Family.objects.filter(guid=family_guid) family = families.get(guid=family_guid) project = family.project - check_project_permissions(project, request.user) + check_family_view_permission(family, request.user) is_analyst = user_is_analyst(request.user) has_case_review_perm = has_case_review_permissions(project, request.user) @@ -122,8 +122,7 @@ def _add_parsed_omims(omims, omim_map, intervals=None): @login_and_policies_required def family_variant_tag_summary(request, family_guid): family = Family.objects.get(guid=family_guid) - project = family.project - check_project_permissions(project, request.user) + check_family_view_permission(family, request.user) response = families_discovery_tags([{'familyGuid': family_guid}]) @@ -135,6 +134,7 @@ def family_variant_tag_summary(request, family_guid): response['familyTagTypeCounts'][family_guid][MME_TAG_NAME] = tags.filter( saved_variants__matchmakersubmissiongenes__isnull=False).values('saved_variants__guid').distinct().count() + project = family.project response['projectsByGuid'] = {project.guid: {}} add_project_tag_types(response['projectsByGuid'], project=project) @@ -230,8 +230,8 @@ def delete_families_handler(request, project_guid): def update_family_fields_handler(request, family_guid): family = Family.objects.get(guid=family_guid) - # check permission - can be edited by anyone with access to the project - check_project_permissions(family.project, request.user) + # check permission - can be edited by anyone with access to the family + check_family_view_permission(family, request.user) request_json = json.loads(request.body) immutable_keys = [] if external_anvil_project_can_edit(family.project, request.user) else ['family_id'] @@ -254,8 +254,8 @@ def _set_display_name(family_json, family_model): @login_and_policies_required def update_family_assigned_analyst(request, family_guid): family = Family.objects.get(guid=family_guid) - # assigned_analyst can be edited by anyone with access to the project - check_project_permissions(family.project, request.user, can_edit=False) + # assigned_analyst can be edited by anyone with access to the family + check_family_view_permission(family, request.user) request_json = json.loads(request.body) assigned_analyst_username = request_json.get('assigned_analyst_username') @@ -281,8 +281,8 @@ def update_family_assigned_analyst(request, family_guid): @login_and_policies_required def update_family_analysed_by(request, family_guid): family = Family.objects.get(guid=family_guid) - # analysed_by can be edited by anyone with access to the project - check_project_permissions(family.project, request.user, can_edit=False) + # analysed_by can be edited by anyone with access to the family + check_family_view_permission(family, request.user) request_json = json.loads(request.body) create_model_from_json(FamilyAnalysedBy, {'family': family, 'data_type': request_json['dataType']}, request.user) @@ -297,7 +297,7 @@ def update_family_pedigree_image(request, family_guid): family = Family.objects.get(guid=family_guid) # check permission - check_project_permissions(family.project, request.user, can_edit=True) + check_project_edit_permission(family.project, request.user) if len(request.FILES) == 0: pedigree_image = None @@ -324,7 +324,7 @@ def update_family_pedigree_image(request, family_guid): def update_family_analysis_groups(request, family_guid): family = Family.objects.get(guid=family_guid) project = family.project - check_project_permissions(project, request.user, can_edit=True) + check_project_edit_permission(project, request.user) request_json = json.loads(request.body) analysis_group_guids = {ag['analysisGroupGuid'] for ag in request_json.get('analysisGroups', [])} @@ -430,7 +430,7 @@ def _get_family_column_map(record): @login_and_policies_required def create_family_note(request, family_guid): family = Family.objects.get(guid=family_guid) - check_project_permissions(family.project, request.user) + check_family_view_permission(family, request.user) return create_note_handler( request, FamilyNote, parent_fields={'family': family}, additional_note_fields=['noteType'], @@ -456,7 +456,7 @@ def delete_family_note(request, family_guid, note_guid): @login_and_policies_required def get_family_rna_seq_data(request, family_guid, gene_id): family = Family.objects.get(guid=family_guid) - check_project_permissions(family.project, request.user) + check_family_view_permission(family, request.user) response = defaultdict(lambda: defaultdict(lambda: {'individualData': {}})) tpm_data = RnaSeqTpm.objects.filter( @@ -483,12 +483,12 @@ def get_family_rna_seq_data(request, family_guid, gene_id): @login_and_policies_required def get_family_phenotype_gene_scores(request, family_guid): - project = Project.objects.get(family__guid=family_guid) - check_project_permissions(project, request.user) + family = Family.objects.get(guid=family_guid) + check_family_view_permission(family, request.user) phenotype_prioritization = get_phenotype_prioritization([family_guid]) gene_ids = {gene_id for indiv in phenotype_prioritization.values() for gene_id in indiv.keys()} return create_json_response({ 'phenotypeGeneScores': phenotype_prioritization, - 'genesById': get_genes_for_variant_display(gene_ids, project.genome_version), + 'genesById': get_genes_for_variant_display(gene_ids, family.project.genome_version), }) diff --git a/seqr/views/apis/igv_api.py b/seqr/views/apis/igv_api.py index fab12d3bea..2fb4becd71 100644 --- a/seqr/views/apis/igv_api.py +++ b/seqr/views/apis/igv_api.py @@ -13,9 +13,9 @@ from seqr.views.utils.json_to_orm_utils import get_or_create_model_from_json from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.orm_to_json_utils import get_json_for_sample -from seqr.views.utils.permissions_utils import get_project_and_check_permissions, external_anvil_project_can_edit, \ +from seqr.views.utils.permissions_utils import get_project_and_check_view_permission, external_anvil_project_can_edit, \ login_and_policies_required, pm_or_data_manager_required, get_project_guids_user_can_view, user_is_data_manager, \ - user_is_pm + user_is_pm, get_project_and_check_edit_permission GS_STORAGE_ACCESS_CACHE_KEY = 'gs_storage_access_cache_entry' GS_STORAGE_URL = 'https://storage.googleapis.com' @@ -92,7 +92,7 @@ def _process_igv_table_handler(parse_uploaded_file, get_valid_matched_individual @pm_or_data_manager_required def receive_igv_table_handler(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) + project = get_project_and_check_edit_permission(project_guid, request.user) def _get_valid_matched_individuals(individual_dataset_mapping): matched_individuals = Individual.objects.filter( @@ -185,7 +185,7 @@ def update_individual_igv_sample(request, individual_guid): @login_and_policies_required def fetch_igv_track(request, project_guid, igv_track_path): - get_project_and_check_permissions(project_guid, request.user) + get_project_and_check_view_permission(project_guid, request.user) if igv_track_path.endswith('.bam.bai') and not does_file_exist(igv_track_path, user=request.user): igv_track_path = igv_track_path.replace('.bam.bai', '.bai') diff --git a/seqr/views/apis/individual_api.py b/seqr/views/apis/individual_api.py index 9d7a0697b8..f17effc715 100644 --- a/seqr/views/apis/individual_api.py +++ b/seqr/views/apis/individual_api.py @@ -24,9 +24,9 @@ GREGOR_FINDING_TAG_TYPE from seqr.views.utils.pedigree_info_utils import parse_pedigree_table, validate_fam_file_records, parse_hpo_terms, \ get_valid_hpo_terms, JsonConstants, ErrorsWarningsException -from seqr.views.utils.permissions_utils import get_project_and_check_permissions, check_project_permissions, \ - get_project_and_check_pm_permissions, login_and_policies_required, has_project_permissions, external_anvil_project_can_edit, \ - pm_or_data_manager_required, check_workspace_perm +from seqr.views.utils.permissions_utils import get_project_and_check_edit_permission, check_project_edit_permission, \ + get_project_and_check_pm_permissions, login_and_policies_required, has_project_edit_permission, external_anvil_project_can_edit, \ + pm_or_data_manager_required, check_workspace_perm, check_family_view_permission from seqr.views.utils.project_context_utils import add_project_tag_type_counts from seqr.views.utils.individual_utils import delete_individuals, add_or_update_individuals_and_families from seqr.views.utils.variant_utils import bulk_create_tagged_variants, get_saved_variant_annotations @@ -57,10 +57,10 @@ def update_individual_handler(request, individual_guid): individual = Individual.objects.get(guid=individual_guid) - project = individual.family.project + family = individual.family - check_project_permissions(project, request.user) - can_edit = has_project_permissions(project, request.user, can_edit=True) + check_family_view_permission(family, request.user) + can_edit = has_project_edit_permission(family.project, request.user) request_json = json.loads(request.body) update_json = request_json if can_edit else {k: v for k, v in request_json.items() if k in {'notes'}} @@ -84,7 +84,7 @@ def update_individual_hpo_terms(request, individual_guid): project = individual.family.project - check_project_permissions(project, request.user, can_edit=True) + check_project_edit_permission(project, request.user) request_json = json.loads(request.body) @@ -422,7 +422,7 @@ def receive_individuals_metadata_handler(request, project_guid): project_guid (string): project GUID """ - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_edit_permission(project_guid, request.user) def process_records(json_records, filename=''): records, errors, warnings = _process_hpo_records(json_records, filename, project, request.user) @@ -651,7 +651,7 @@ def save_individuals_metadata_table_handler(request, project_guid, upload_file_i """ Handler for 'save' requests to apply HPO terms tables previously uploaded through receive_individuals_metadata_handler """ - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_edit_permission(project_guid, request.user) json_records, _ = load_uploaded_file(upload_file_id) @@ -698,7 +698,7 @@ def save_individuals_metadata_table_handler(request, project_guid, upload_file_i def import_gregor_metadata(request, project_guid): request_json = json.loads(request.body) sample_type = request_json.get('sampleType', 'genome') - project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) + project = get_project_and_check_edit_permission(project_guid, request.user) workspace_meta = check_workspace_perm( request.user, CAN_VIEW, request_json['workspaceNamespace'], request_json['workspaceName'], meta_fields=['workspace.bucketName'] @@ -883,8 +883,8 @@ def _parse_participant_val(column, value, participant_sample_lookup): @login_and_policies_required def get_individual_rna_seq_data(request, individual_guid): individual = Individual.objects.get(guid=individual_guid) - project = individual.family.project - check_project_permissions(project, request.user) + family = individual.family + check_family_view_permission(family, request.user) filters = {'sample__individual': individual} outlier_data = get_json_for_rna_seq_outliers(filters, significant_only=False, individual_guid=individual_guid) @@ -892,7 +892,7 @@ def get_individual_rna_seq_data(request, individual_guid): genes_to_show = get_genes({ gene_id for rna_data in outlier_data.get(individual_guid, {}).values() for gene_id, data in rna_data.items() if any([d['isSignificant'] for d in (data if isinstance(data, list) else [data])]) - }, genome_version=project.genome_version) + }, genome_version=family.project.genome_version) return create_json_response({ 'rnaSeqData': outlier_data, diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 910478d944..479231a8cc 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -1060,7 +1060,7 @@ def _is_expected_individuals_metadata_upload(self, response, expected_families=F def test_individuals_metadata_table_handler(self): url = reverse(receive_individuals_metadata_handler, args=['R0001_1kg']) - self.check_collaborator_login(url) + self.check_manager_login(url) # Send invalid requests header = 'family_id,indiv_id,hpo_term_yes,hpo_term_no' @@ -1106,7 +1106,7 @@ def test_individuals_metadata_table_handler(self): read_tmp_table_logs = self._read_tmp_table_logs('6aea3d1f1bfc295340aa504370102fd5') offset = 2 if read_tmp_table_logs else 1 self.assert_json_logs(None, read_tmp_table_logs, offset=offset) - self.assert_json_logs(self.collaborator_user, [ + self.assert_json_logs(self.manager_user, [ ('update Individual I000002_na19678', {'dbUpdate': mock.ANY}), ('update Individual I000001_na19675', {'dbUpdate': mock.ANY}), ('Reloading dictionary seqrdb_individual_metadata_dict', None), @@ -1114,7 +1114,7 @@ def test_individuals_metadata_table_handler(self): def test_individuals_metadata_hpo_term_number_table_handler(self): url = reverse(receive_individuals_metadata_handler, args=['R0001_1kg']) - self.check_collaborator_login(url) + self.check_manager_login(url) header = 'family_id,individual_id,affected,hpo_number,hpo_number,sex,birth,other affected relatives,onset,expected inheritance,maternal ancestry,candidate genes' rows = [ diff --git a/seqr/views/apis/locus_list_api.py b/seqr/views/apis/locus_list_api.py index 625de636c5..f8c2e3dda3 100644 --- a/seqr/views/apis/locus_list_api.py +++ b/seqr/views/apis/locus_list_api.py @@ -12,8 +12,9 @@ create_model_from_json from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.orm_to_json_utils import get_json_for_locus_lists, get_json_for_locus_list -from seqr.views.utils.permissions_utils import get_project_and_check_permissions, check_locus_list_permissions, \ - check_user_created_object_permissions, login_and_policies_required, get_project_guids_user_can_view +from seqr.views.utils.permissions_utils import get_project_and_check_view_permission, check_locus_list_permissions, \ + check_user_created_object_permissions, login_and_policies_required, get_project_guids_user_can_view, \ + get_project_and_check_edit_permission logger = SeqrLogger(__name__) @@ -135,7 +136,7 @@ def delete_locus_list_handler(request, locus_list_guid): @login_and_policies_required def add_project_locus_lists(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_view_permission(project_guid, request.user) request_json = json.loads(request.body) locus_lists = LocusList.objects.filter(guid__in=request_json['locusListGuids']) for locus_list in locus_lists: @@ -151,7 +152,7 @@ def add_project_locus_lists(request, project_guid): @login_and_policies_required def delete_project_locus_lists(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) + project = get_project_and_check_edit_permission(project_guid, request.user) request_json = json.loads(request.body) locus_lists = LocusList.objects.filter(guid__in=request_json['locusListGuids']) for locus_list in locus_lists: diff --git a/seqr/views/apis/project_api.py b/seqr/views/apis/project_api.py index 3585ae625e..4f0669ca6a 100644 --- a/seqr/views/apis/project_api.py +++ b/seqr/views/apis/project_api.py @@ -22,7 +22,7 @@ get_json_for_project_collaborator_list, get_json_for_matchmaker_submissions, \ get_json_for_family_notes, _get_json_for_individuals, get_json_for_project_collaborator_groups, \ FAMILY_ADDITIONAL_VALUES -from seqr.views.utils.permissions_utils import get_project_and_check_permissions, check_project_permissions, \ +from seqr.views.utils.permissions_utils import get_project_and_check_view_permission, get_project_and_check_edit_permission, \ check_user_created_object_permissions, pm_required, user_is_pm, login_and_policies_required, \ has_workspace_perm, has_case_review_permissions, is_internal_anvil_project, get_project_and_check_pm_permissions, \ check_project_pm_permission, user_is_data_manager, external_anvil_project_can_edit @@ -77,9 +77,7 @@ def _is_valid_anvil_workspace(request_json, user): @login_and_policies_required def update_project_handler(request, project_guid): - project = Project.objects.get(guid=project_guid) - - check_project_permissions(project, request.user, can_edit=True) + project = get_project_and_check_edit_permission(project_guid, request.user) request_json = json.loads(request.body) updated_fields = set() @@ -106,7 +104,7 @@ def update_project_workspace(request, project_guid): if not is_anvil_authenticated(request.user): raise PermissionDenied() - project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) + project = get_project_and_check_edit_permission(project_guid, request.user) request_json = json.loads(request.body) if not _is_valid_anvil_workspace(request_json, request.user): @@ -131,7 +129,7 @@ def delete_project_handler(request, project_guid): @login_and_policies_required def project_page_data(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_view_permission(project_guid, request.user) update_project_from_json(project, {'last_accessed_date': timezone.now()}, request.user) return create_json_response({ 'projectsByGuid': { @@ -171,7 +169,7 @@ def _get_formatted_value(value, config, *args): @login_and_policies_required def project_families(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_view_permission(project_guid, request.user) family_models = Family.objects.filter(project=project) families = family_models.values( @@ -215,7 +213,7 @@ def project_families(request, project_guid): @login_and_policies_required def project_overview(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_view_permission(project_guid, request.user) datasets = Dataset.objects.filter( Q(active_individuals__family__project=project) | Q(inactive_individuals__family__project=project) @@ -252,7 +250,7 @@ def project_overview(request, project_guid): @login_and_policies_required def project_collaborators(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_view_permission(project_guid, request.user) return create_json_response({ 'projectsByGuid': {project_guid: { @@ -264,7 +262,7 @@ def project_collaborators(request, project_guid): @login_and_policies_required def project_individuals(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_view_permission(project_guid, request.user) individuals = _get_json_for_individuals( Individual.objects.filter(family__project=project), user=request.user, project_guid=project_guid, add_hpo_details=True, has_case_review_perm=has_case_review_permissions(project, request.user)) @@ -276,7 +274,7 @@ def project_individuals(request, project_guid): @login_and_policies_required def project_analysis_groups(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_view_permission(project_guid, request.user) return create_json_response({ 'analysisGroupsByGuid': get_project_analysis_groups([project], project_guid) @@ -285,7 +283,7 @@ def project_analysis_groups(request, project_guid): @login_and_policies_required def project_locus_lists(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_view_permission(project_guid, request.user) locus_list_json, _ = get_project_locus_lists([project], request.user, include_metadata=True) return create_json_response({ @@ -296,7 +294,7 @@ def project_locus_lists(request, project_guid): @login_and_policies_required def project_family_notes(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_view_permission(project_guid, request.user) family_notes = get_json_for_family_notes(FamilyNote.objects.filter(family__project=project), is_analyst=False) return create_json_response({ @@ -305,7 +303,7 @@ def project_family_notes(request, project_guid): @login_and_policies_required def project_mme_submisssions(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_view_permission(project_guid, request.user) models = MatchmakerSubmission.objects.filter( individual__family__project=project).prefetch_related('matchmakersubmissiongenes_set') @@ -325,7 +323,7 @@ def project_mme_submisssions(request, project_guid): @login_and_policies_required def project_notifications(request, project_guid, read_status): - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_view_permission(project_guid, request.user) is_subscriber = project.subscribers.user_set.filter(id=request.user.id).exists() if not is_subscriber: max_loaded = _project_notifications( @@ -360,14 +358,14 @@ def _project_notifications(project, notifications): @login_and_policies_required def mark_read_project_notifications(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_view_permission(project_guid, request.user) _project_notifications(project, request.user.notifications).mark_all_as_read() return create_json_response({'readCount': request.user.notifications.read().count(), 'unreadNotifications': []}) @login_and_policies_required def subscribe_project_notifications(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_view_permission(project_guid, request.user) request.user.groups.add(project.subscribers) return create_json_response({'isSubscriber': True}) diff --git a/seqr/views/apis/project_categories_api.py b/seqr/views/apis/project_categories_api.py index 55f15ea3c2..e0804dafc4 100644 --- a/seqr/views/apis/project_categories_api.py +++ b/seqr/views/apis/project_categories_api.py @@ -1,18 +1,15 @@ import json -from seqr.models import Project, ProjectCategory +from seqr.models import ProjectCategory from seqr.views.utils.json_to_orm_utils import create_model_from_json from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.orm_to_json_utils import _get_json_for_project -from seqr.views.utils.permissions_utils import check_project_permissions, login_and_policies_required +from seqr.views.utils.permissions_utils import get_project_and_check_edit_permission, login_and_policies_required @login_and_policies_required def update_project_categories_handler(request, project_guid): - project = Project.objects.get(guid=project_guid) - - # check permissions - check_project_permissions(project, request.user, can_edit=True) + project = get_project_and_check_edit_permission(project_guid, request.user) request_json = json.loads(request.body) diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index 3d2589a250..d619525d28 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -2,6 +2,9 @@ from datetime import datetime, timedelta from django.db.models import Count, Q, F, Value +from django.db.models.expressions import Case, When +from django.db.models.functions import TruncDate +from django.contrib.auth.models import User from django.contrib.postgres.aggregates import ArrayAgg import json import re @@ -16,10 +19,10 @@ from seqr.views.utils.anvil_metadata_utils import parse_anvil_metadata, anvil_export_airtable_fields, \ FAMILY_ROW_TYPE, SUBJECT_ROW_TYPE, SAMPLE_ROW_TYPE, DISCOVERY_ROW_TYPE, PARTICIPANT_TABLE, PHENOTYPE_TABLE, \ EXPERIMENT_TABLE, EXPERIMENT_LOOKUP_TABLE, FINDINGS_TABLE, GENE_COLUMN, FAMILY_INDIVIDUAL_FIELDS -from seqr.views.utils.export_utils import export_multiple_files, write_multiple_files +from seqr.views.utils.export_utils import export_multiple_files, write_multiple_files, export_table from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.orm_to_json_utils import get_json_for_queryset -from seqr.views.utils.permissions_utils import user_is_analyst, get_project_and_check_permissions, \ +from seqr.views.utils.permissions_utils import user_is_analyst, get_project_and_check_view_permission, \ get_project_guids_user_can_view, get_internal_projects, pm_or_analyst_required, active_user_has_policies_and_passes_test from seqr.views.utils.terra_api_utils import anvil_enabled from seqr.views.utils.variant_utils import DISCOVERY_CATEGORY @@ -39,55 +42,115 @@ @pm_or_analyst_required def seqr_stats(request): - non_demo_projects = Project.objects.filter(is_demo=False) + project_qs, dataset_qs_list = _get_project_aggregated_qs(additional_model=(Project, '')) - project_models = { - 'demo': Project.objects.filter(is_demo=True), - } - if anvil_enabled(): - is_anvil_q = Q(workspace_namespace='') | Q(workspace_namespace__isnull=True) - anvil_projects = non_demo_projects.exclude(is_anvil_q) - internal_ids = get_internal_projects().values_list('id', flat=True) - project_models.update({ - 'internal': anvil_projects.filter(id__in=internal_ids), - 'external': anvil_projects.exclude(id__in=internal_ids), - 'no_anvil': non_demo_projects.filter(is_anvil_q), - }) - else: - project_models.update({ - 'non_demo': non_demo_projects, - }) + grouped_sample_counts = defaultdict(lambda: defaultdict(int)) + for qs in dataset_qs_list: + for agg in qs: + grouped_sample_counts[f"{agg['sample_type']}__{agg['dataset_type']}"][_agg_key(agg)] += agg['count'] - grouped_sample_counts = defaultdict(dict) - for project_key, projects in project_models.items(): - samples_counts = _get_sample_counts( - Dataset.objects.filter(active_individuals__family__project__in=projects), - count=Count('active_individuals'), - ) - samples_counts.update(_get_sample_counts( - RnaSample.objects.filter(individual__family__project__in=projects, is_active=True).annotate(sample_type=Value('RNA')), - data_type_key='data_type', count=Count('*')) - ) - for k, v in samples_counts.items(): - grouped_sample_counts[k][project_key] = v + project_aggs = project_qs.annotate( + count=Count('id', distinct=True), + family_count=Count('family', distinct=True), + individual_count=Count('family__individual', distinct=True), + ) + project_counts = defaultdict(int) + families_count = defaultdict(int) + individuals_count = defaultdict(int) + for agg in project_aggs: + key = _agg_key(agg) + project_counts[key] += agg['count'] + families_count[key] += agg['family_count'] + individuals_count[key] += agg['individual_count'] + + now = datetime.now() + user_counts = User.objects.filter(is_active=True).aggregate( + total=Count('id'), + multipleLogins=Count('id', filter=Q(last_login__isnull=False) & ~Q(last_login=F('date_joined'))), + lastMonth=Count('id', filter=Q(last_login__gte=now - timedelta(days=30))), + lastYear=Count('id', filter=Q(last_login__gte=now - timedelta(days=365))), + thisYear=Count('id', filter=Q(last_login__gte=datetime(now.year, 1, 1))), + ) return create_json_response({ - 'projectsCount': {k: projects.count() for k, projects in project_models.items()}, - 'familiesCount': { - k: Family.objects.filter(project__in=projects).count() for k, projects in project_models.items() - }, - 'individualsCount': { - k: Individual.objects.filter(family__project__in=projects).count() for k, projects in project_models.items() - }, + 'projectsCount': project_counts, + 'familiesCount': families_count, + 'individualsCount': individuals_count, + 'usersCounts': user_counts, 'sampleCountsByType': grouped_sample_counts, }) -def _get_sample_counts(sample_q, count=None, data_type_key='dataset_type'): - samples_agg = sample_q.values('sample_type', data_type_key).annotate(count=count) - return { - f'{sample_agg["sample_type"]}__{sample_agg[data_type_key]}': sample_agg['count'] for sample_agg in samples_agg - } +def _get_project_aggregated_qs(additional_model=None, additional_aggs=None): + prefixes = ['active_individuals__family__project__', 'individual__family__project__'] + if additional_model: + prefixes.append(additional_model[1]) + agg_fields = [{'demo': F(f'{prefix}is_demo')} for prefix in prefixes] + if anvil_enabled(): + internal_ids = get_internal_projects().values_list('id', flat=True) + for i, prefix in enumerate(prefixes): + agg_fields[i].update({ + 'no_anvil': Q(**{f'{prefix}workspace_namespace': ''}) | Q(**{f'{prefix}workspace_namespace__isnull': True}), + 'is_internal': Case(When(**{f'{prefix}id__in': internal_ids, 'then': Value(True)}), default=Value(False)), + }) + + dataset_qs = [ + models.annotate(**agg_fields[i], **(additional_aggs or {})).filter(demo__isnull=False).values( + 'sample_type', 'dataset_type', *agg_fields[i], *(additional_aggs or []), + ).annotate(count=Count('active_individuals')) for i, models in enumerate([ + Dataset.objects, RnaSample.objects.filter(is_active=True).annotate( + sample_type=Value('RNA'), dataset_type=F('data_type'), active_individuals=F('individual_id'), loaded_date=F('created_date'), + ), + ]) + ] + + if additional_model: + fields = agg_fields[-1] + return additional_model[0].objects.annotate(**fields).values(*fields), dataset_qs + + return dataset_qs + + +def _agg_key(agg): + if agg['demo']: + return 'demo' + elif agg.get('no_anvil'): + return 'no_anvil' + elif 'is_internal' in agg: + return 'internal' if agg['is_internal'] else 'external' + return 'non_demo' + + +HEADER = ['date_loaded', 'num_samples', 'sample_type', 'dataset_type', 'is_demo'] +DATASET_TYPE_LOOKUP = { + **RnaSample.DATA_TYPE_LOOKUP, + **{k: v.split(' ')[0] for k, v in Dataset.DATASET_TYPE_LOOKUP.items()}, +} + + +@pm_or_analyst_required +def sample_stats_download(request): + dataset_qs_list = _get_project_aggregated_qs(additional_aggs={'loaded': TruncDate('loaded_date')}) + rows = sorted( + [_format_export_row(**item) for qs in dataset_qs_list for item in qs], + key=lambda row: (row[0], -row[1], *row[2:]), + ) + + header = HEADER + if anvil_enabled(): + header = header + ['anvil_status'] + + file_format = request.GET.get('file_format', 'tsv') + return export_table('seqr_sample_loading', header, rows, file_format) + + +def _format_export_row(loaded, count, sample_type, dataset_type, demo, no_anvil='', is_internal=''): + row = [loaded.strftime('%Y-%m-%d'), count, sample_type, DATASET_TYPE_LOOKUP.get(dataset_type, 'Unknown'), demo] + if no_anvil: + row.append('No AnVIL') + elif is_internal != '': + row.append('Internal' if is_internal else 'External') + return row # AnVIL metadata @@ -122,7 +185,7 @@ def _get_sample_counts(sample_q, count=None, data_type_key='dataset_type'): @airtable_enabled_analyst_required def anvil_export(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_view_permission(project_guid, request.user) parsed_rows = defaultdict(list) family_id_map = {} @@ -968,7 +1031,7 @@ def _get_metadata_projects(project_guid, user): return get_internal_projects().filter(guid__in=get_project_guids_user_can_view(user)) if project_guid == GREGOR_CATEGORY.lower(): return Project.objects.filter(projectcategory__name=GREGOR_CATEGORY) - return [get_project_and_check_permissions(project_guid, user)] + return [get_project_and_check_view_permission(project_guid, user)] ANALYSIS_DATA_TYPE_LOOKUP = dict(FamilyAnalysedBy.DATA_TYPE_CHOICES) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 3e46cf9680..50492f9214 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -5,7 +5,7 @@ from settings import AIRTABLE_URL from seqr.models import Project, RnaSample -from seqr.views.apis.report_api import seqr_stats, anvil_export, gregor_export, family_metadata, variant_metadata +from seqr.views.apis.report_api import seqr_stats, sample_stats_download, anvil_export, gregor_export, family_metadata, variant_metadata from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest @@ -683,11 +683,30 @@ def test_seqr_stats(self): response = self.client.get(url) self.assertEqual(response.status_code, 200) response_json = response.json() - self.assertSetEqual(set(response_json.keys()), {'projectsCount', 'individualsCount', 'familiesCount', 'sampleCountsByType'}) + self.assertSetEqual(set(response_json.keys()), {'projectsCount', 'individualsCount', 'familiesCount', 'sampleCountsByType', 'usersCounts'}) self.assertDictEqual(response_json['projectsCount'], self.STATS_DATA['projectsCount']) self.assertDictEqual(response_json['individualsCount'], self.STATS_DATA['individualsCount']) self.assertDictEqual(response_json['familiesCount'], self.STATS_DATA['familiesCount']) self.assertDictEqual(response_json['sampleCountsByType'], self.STATS_DATA['sampleCountsByType']) + self.assertDictEqual(response_json['usersCounts'], { + 'total': 10, 'multipleLogins': 7, 'lastMonth': 5, 'lastYear': 5, 'thisYear': 5, + }) + + self.check_no_analyst_no_access(url, has_override=self.HAS_PM_OVERRIDE) + + def test_sample_stats_download(self): + no_access_project = Project.objects.get(id=3) + no_access_project.workspace_namespace = None + no_access_project.save() + + url = reverse(sample_stats_download) + self.check_analyst_login(url) + + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertListEqual( + [line.split('\t') for line in response.content.decode().strip('\n').split('\n')], self.EXPORT_DATA, + ) self.check_no_analyst_no_access(url, has_override=self.HAS_PM_OVERRIDE) @@ -1557,6 +1576,17 @@ class LocalReportAPITest(AuthenticationTestCase, ReportAPITest): 'RNA__E': {'non_demo': 1}, }, } + EXPORT_DATA = [ + ['date_loaded', 'num_samples', 'sample_type', 'dataset_type', 'is_demo'], + ['2017-02-05', '7', 'WES', 'Variant', 'False'], + ['2017-02-05', '3', 'RNA', 'Splice Outlier', 'False'], + ['2017-02-05', '2', 'RNA', 'TPM', 'False'], + ['2017-02-05', '1', 'RNA', 'Expression Outlier', 'False'], + ['2018-02-05', '3', 'WES', 'SV', 'False'], + ['2018-02-05', '3', 'WGS', 'SV', 'False'], + ['2020-02-05', '1', 'WGS', 'Variant', 'True'], + ['2022-02-05', '1', 'WES', 'Mitochondria', 'False'], + ] def _check_anvil_export_response(self, response, *args): self.assertEqual(response.status_code, 403) @@ -1584,3 +1614,15 @@ class AnvilReportAPITest(AnvilAuthenticationTestCase, ReportAPITest): 'RNA__E': {'internal': 1}, }, } + EXPORT_DATA = [ + ['date_loaded', 'num_samples', 'sample_type', 'dataset_type', 'is_demo', 'anvil_status'], + ['2017-02-05', '7', 'WES', 'Variant', 'False', 'Internal'], + ['2017-02-05', '2', 'RNA', 'Splice Outlier', 'False', 'Internal'], + ['2017-02-05', '2', 'RNA', 'TPM', 'False', 'Internal'], + ['2017-02-05', '1', 'RNA', 'Expression Outlier', 'False', 'Internal'], + ['2017-02-05', '1', 'RNA', 'Splice Outlier', 'False', 'External'], + ['2018-02-05', '3', 'WES', 'SV', 'False', 'Internal'], + ['2018-02-05', '3', 'WGS', 'SV', 'False', 'External'], + ['2020-02-05', '1', 'WGS', 'Variant', 'True', 'No AnVIL'], + ['2022-02-05', '1', 'WES', 'Mitochondria', 'False', 'Internal'], + ] diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 68f99caf28..27f8761597 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -5,15 +5,15 @@ from clickhouse_search.models.postgres_dicts import DiscoveryVariantDict, ExcludedVariantDict from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\ - Family, GeneNote, Project, Dataset + Family, GeneNote, Dataset from seqr.utils.xpos_utils import get_xpos from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.orm_to_json_utils import get_json_for_saved_variants_with_tags, get_json_for_variant_note, \ get_json_for_saved_variants_child_entities, get_json_for_gene_notes_by_gene_id, STRUCTURED_METADATA_TAG_TYPES -from seqr.views.utils.permissions_utils import get_project_and_check_permissions, check_project_permissions, \ - login_and_policies_required +from seqr.views.utils.permissions_utils import get_project_and_check_view_permission, check_project_edit_permission, \ + login_and_policies_required, check_family_view_permission, check_families_view_permission from seqr.views.utils.variant_utils import get_variants_response, parse_saved_variant_json, DISCOVERY_CATEGORY @@ -23,7 +23,7 @@ @login_and_policies_required def saved_variant_data(request, project_guid, variant_guids=None): - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_view_permission(project_guid, request.user) family_guids = request.GET['families'].split(',') if request.GET.get('families') else None variant_guids = variant_guids.split(',') if variant_guids else None @@ -51,7 +51,7 @@ def saved_variant_data(request, project_guid, variant_guids=None): @login_and_policies_required def create_manual_saved_variant_handler(request, family_guid): family = Family.objects.get(guid=family_guid) - check_project_permissions(family.project, request.user) + check_family_view_permission(family, request.user) variant_json = json.loads(request.body) tags = variant_json.pop('tags', []) @@ -98,7 +98,7 @@ def create_saved_variant_handler(request): family_guid = variant_json['familyGuid'] family = Family.objects.get(guid=family_guid) - check_project_permissions(family.project, request.user) + check_family_view_permission(family, request.user) variants_json = variant_json['variant'] if not isinstance(variant_json['variant'], list): @@ -130,7 +130,7 @@ def create_variant_note_handler(request, variant_guids): family_guid = request_json.pop('familyGuid') family = Family.objects.get(guid=family_guid) - check_project_permissions(family.project, request.user) + check_family_view_permission(family, request.user) all_variant_guids = variant_guids.split(',') saved_variants = SavedVariant.objects.filter(guid__in=all_variant_guids) @@ -180,9 +180,8 @@ def _create_variant_note(saved_variants, note_json, user): @login_and_policies_required def update_variant_note_handler(request, variant_guids, note_guid): note = VariantNote.objects.get(guid=note_guid) - projects = {saved_variant.family.project for saved_variant in note.saved_variants.all()} - for project in projects: - check_project_permissions(project, request.user) + families = Family.objects.filter(id__in=note.saved_variants.values_list('family_id', flat=True)) + check_families_view_permission(families, request.user) request_json = json.loads(request.body) update_model_from_json(note, request_json, user=request.user, allow_unknown_keys=True) @@ -198,9 +197,8 @@ def update_variant_note_handler(request, variant_guids, note_guid): def delete_variant_note_handler(request, variant_guids, note_guid): variant_guids = variant_guids.split(',') note = VariantNote.objects.get(guid=note_guid) - projects = {saved_variant.family.project for saved_variant in note.saved_variants.all()} - for project in projects: - check_project_permissions(project, request.user) + families = Family.objects.filter(id__in=note.saved_variants.values_list('family_id', flat=True)) + check_families_view_permission(families, request.user) note.delete_model(request.user, user_can_delete=True) saved_variants_by_guid = {} @@ -238,7 +236,7 @@ def update_variant_acmg_classification_handler(request, variant_guid): def _update_variant_acmg_classification(request, variant_guid): saved_variant = SavedVariant.objects.get(guid=variant_guid) - check_project_permissions(saved_variant.family.project, request.user) + check_family_view_permission(saved_variant.family, request.user) request_json = json.loads(request.body) variant = request_json.get('variant') @@ -263,8 +261,8 @@ def _update_variant_tag_models(request, variant_guids, tag_key, model_cls, get_t request_json = json.loads(request.body) family_guid = request_json.pop('familyGuid') - project = Project.objects.get(family__guid=family_guid) - check_project_permissions(project, request.user) + family = Family.objects.get(guid=family_guid) + check_family_view_permission(family, request.user) all_variant_guids = set(variant_guids.split(',')) saved_variants = SavedVariant.objects.filter(guid__in=all_variant_guids) @@ -357,7 +355,7 @@ def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=Vari @login_and_policies_required def update_variant_main_transcript(request, variant_guid, transcript_id): saved_variant = SavedVariant.objects.get(guid=variant_guid) - check_project_permissions(saved_variant.family.project, request.user, can_edit=True) + check_project_edit_permission(saved_variant.family.project, request.user) update_model_from_json(saved_variant, { 'selected_main_transcript_id': transcript_id, diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index c13cca28f2..6e560f2c00 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -786,7 +786,10 @@ def test_create_update_and_delete_variant_note(self): new_variant_note = VariantNote.objects.filter(guid=updated_note_response['noteGuid']) self.assertEqual(len(new_variant_note), 0) - self.assert_no_list_ws_has_al(8) + self.mock_list_workspaces.assert_called_with(self.collaborator_user) + self.assertEqual(self.mock_list_workspaces.call_count, 2) + self.assertEqual(self.mock_get_ws_access_level.call_count, 6) + self.assert_no_extra_anvil_calls() def test_create_partially_saved_compound_het_variant_note(self): # compound het 5 is not saved, whereas compound het 1 is saved @@ -940,7 +943,10 @@ def test_create_update_and_delete_compound_hets_variant_note(self): variants = SavedVariant.objects.filter(guid__in=[COMPOUND_HET_1_GUID, COMPOUND_HET_2_GUID]) self.assertEqual(len(variants), 0) - self.assert_no_list_ws_has_al(7) + self.mock_list_workspaces.assert_called_with(self.collaborator_user) + self.assertEqual(self.mock_list_workspaces.call_count, 3) + self.assertEqual(self.mock_get_ws_access_level.call_count, 4) + self.assert_no_extra_anvil_calls() def test_update_variant_tags(self): variant_tags = VariantTag.objects.filter(saved_variants__guid__contains=VARIANT_GUID) diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 6316d0167b..50d7647b7a 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -22,7 +22,7 @@ from seqr.views.utils.orm_to_json_utils import get_json_for_matchmaker_submissions, get_json_for_saved_variants,\ add_individual_hpo_details, INDIVIDUAL_DISPLAY_NAME_EXPR, AIP_TAG_TYPE from seqr.views.utils.permissions_utils import analyst_required, user_is_analyst, get_project_guids_user_can_view, \ - login_and_policies_required, get_project_and_check_permissions, get_internal_projects + login_and_policies_required, get_project_and_check_view_permission, get_internal_projects from seqr.views.utils.anvil_metadata_utils import parse_anvil_metadata, anvil_export_airtable_fields, FAMILY_ROW_TYPE, SUBJECT_ROW_TYPE, DISCOVERY_ROW_TYPE from seqr.views.utils.variant_utils import get_variants_response, bulk_create_tagged_variants, get_saved_variant_annotations, DISCOVERY_CATEGORY from settings import SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL @@ -305,7 +305,7 @@ def _get_metadata_projects(request, project_guid): raise PermissionDenied() projects = Project.objects.filter(projectcategory__name__iexact=GREGOR_CATEGORY) else: - projects = [get_project_and_check_permissions(project_guid, request.user)] + projects = [get_project_and_check_view_permission(project_guid, request.user)] return projects, include_airtable diff --git a/seqr/views/apis/users_api.py b/seqr/views/apis/users_api.py index 8e21cee8c2..ed258a894c 100644 --- a/seqr/views/apis/users_api.py +++ b/seqr/views/apis/users_api.py @@ -14,8 +14,9 @@ from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.orm_to_json_utils import get_json_for_user, get_json_for_project_collaborator_list, \ get_project_collaborators_by_username, get_json_for_project_collaborator_groups, PROJECT_ACCESS_GROUP_NAMES -from seqr.views.utils.permissions_utils import get_project_guids_user_can_view, get_project_and_check_permissions, \ - login_and_policies_required, login_active_required, active_user_has_policies_and_passes_test +from seqr.views.utils.permissions_utils import get_project_guids_user_can_view, get_project_and_check_view_permission, \ + login_and_policies_required, login_active_required, active_user_has_policies_and_passes_test, \ + get_project_and_check_edit_permission from seqr.views.utils.terra_api_utils import oauth_enabled, anvil_enabled from settings import BASE_URL, SEQR_TOS_VERSION, SEQR_PRIVACY_VERSION @@ -48,7 +49,7 @@ def get_all_user_group_options(request): @login_and_policies_required def get_project_collaborator_options(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user) + project = get_project_and_check_view_permission(project_guid, request.user) user_fields = {'display_name', 'username', 'email'} users = get_project_collaborators_by_username( request.user, project, fields=user_fields, expand_user_groups=True, @@ -136,7 +137,7 @@ def update_policies(request): @require_anvil_disabled def create_project_collaborator(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) + project = get_project_and_check_edit_permission(project_guid, request.user) request_json = json.loads(request.body) if not request_json.get('email'): @@ -185,7 +186,7 @@ def _update_existing_user(user, project, request_json): @require_anvil_disabled def update_project_collaborator(request, project_guid, username): - project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) + project = get_project_and_check_edit_permission(project_guid, request.user) user = User.objects.get(username=username) request_json = json.loads(request.body) @@ -194,7 +195,7 @@ def update_project_collaborator(request, project_guid, username): @require_anvil_disabled def delete_project_collaborator(request, project_guid, username): - project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) + project = get_project_and_check_edit_permission(project_guid, request.user) user = User.objects.get(username=username) project.can_view_group.user_set.remove(user) @@ -207,7 +208,7 @@ def delete_project_collaborator(request, project_guid, username): @require_anvil_disabled def update_project_collaborator_group(request, project_guid, name): - project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) + project = get_project_and_check_edit_permission(project_guid, request.user) group = Group.objects.get(name=name) request_json = json.loads(request.body) @@ -224,7 +225,7 @@ def update_project_collaborator_group(request, project_guid, name): @require_anvil_disabled def delete_project_collaborator_group(request, project_guid, name): - project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) + project = get_project_and_check_edit_permission(project_guid, request.user) group = Group.objects.get(name=name) remove_perm(user_or_group=group, perm=CAN_VIEW, obj=project) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index ded007b9a8..3a00b5a078 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -28,8 +28,8 @@ create_model_from_json from seqr.views.utils.orm_to_json_utils import get_json_for_saved_variants_with_tags, get_json_for_saved_search,\ get_json_for_saved_searches, FAMILY_ADDITIONAL_VALUES -from seqr.views.utils.permissions_utils import check_project_permissions, get_project_guids_user_can_view, \ - login_and_policies_required, check_user_created_object_permissions, check_projects_view_permission, user_is_analyst +from seqr.views.utils.permissions_utils import check_family_view_permission, get_project_guids_user_can_view, \ + login_and_policies_required, check_user_created_object_permissions, check_families_view_permission, user_is_analyst from seqr.views.utils.project_context_utils import get_projects_child_entities from seqr.views.utils.variant_utils import get_variants_response, variant_dataset_type from seqr.views.utils.vlm_utils import vlm_lookup @@ -173,7 +173,7 @@ def _get_exclude_keys(search_hash, user): def query_single_variant_handler(request, variant_id): families = Family.objects.filter(guid=request.GET.get('familyGuid')) family = families.first() - check_project_permissions(family.project, request.user) + check_family_view_permission(family, request.user) variants = get_clickhouse_variants(families, request.user, raw_variant_items=variant_id, variant_ids=[variant_id]) if not variants: @@ -276,7 +276,7 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo @login_and_policies_required def get_variant_gene_breakdown(request, search_hash): results_model = VariantSearchResults.objects.get(search_hash=search_hash) - projects = _check_results_permission(results_model, request.user) + genome_version = _check_results_permission(results_model, request.user) results = _get_variants_with_cache(_get_search_cache_key, _query_variants, results_model, request.user) flat_variants = [ @@ -287,12 +287,12 @@ def get_variant_gene_breakdown(request, search_hash): gene_ids = var['transcripts'].keys() if 'transcripts' in var else {t['geneId'] for t in var['sortedTranscriptConsequences']} for gene_id in gene_ids: gene_counts[gene_id]['total'] += 1 - for family_guid in var['familyGuids']: + for family_guid in var.get('familyGuids', []): gene_counts[gene_id]['families'][family_guid] += 1 return create_json_response({ 'searchGeneBreakdown': {search_hash: gene_counts}, - 'genesById': get_genes_for_variant_display(list(gene_counts.keys()), projects.first().genome_version), + 'genesById': get_genes_for_variant_display(list(gene_counts.keys()), genome_version), }) @@ -392,29 +392,30 @@ def search_context_handler(request): response = _get_saved_searches(request.user) context = json.loads(request.body) - projects = None + families = None if context.get('projectGuid'): - projects = Project.objects.filter(guid=context.get('projectGuid')) + families = Family.objects.filter(project__guid=context.get('projectGuid')) elif context.get('familyGuid'): - projects = Project.objects.filter(family__guid=context.get('familyGuid')) + families = Family.objects.filter(guid=context.get('familyGuid')) elif context.get('analysisGroupGuid'): - projects = Project.objects.filter(analysisgroup__guid=context.get('analysisGroupGuid')) + families = Family.objects.filter(analysisgroup__guid=context.get('analysisGroupGuid')) elif context.get('projectCategoryGuid'): - projects = Project.objects.filter(projectcategory__guid=context.get('projectCategoryGuid')) + families = Family.objects.filter(project__projectcategory__guid=context.get('projectCategoryGuid')) elif context.get('searchHash'): search_context = context.get('searchParams') try: results_model = _get_or_create_results_model(context['searchHash'], search_context, request.user) except Exception as e: return create_json_response({'error': str(e)}, status=400, reason=str(e)) - projects = Project.objects.filter(family__in=results_model.families.all()).distinct() + families = results_model.families.all() - if not projects: + if not families: error = 'Invalid context params: {}'.format(json.dumps(context)) return create_json_response({'error': error}, status=400, reason=error) - check_projects_view_permission(projects, request.user) + check_families_view_permission(families, request.user) + projects = Project.objects.filter(family__in=families).distinct() project_guid = projects[0].guid if len(projects) == 1 else None response.update(get_projects_child_entities(projects, project_guid, request.user)) @@ -511,13 +512,14 @@ def delete_saved_search_handler(request, saved_search_guid): def _check_results_permission(results_model, user, project_perm_check=None): - projects = Project.objects.filter(family__variantsearchresults=results_model) - check_projects_view_permission(projects, user) + families = Family.objects.filter(variantsearchresults=results_model) + check_families_view_permission(families, user) if project_perm_check: - for project in projects: + for project in {f.project for f in families.prefetch_related('project')}: if not project_perm_check(project): raise PermissionDenied() - return projects + family = families.first() + return family.project.genome_version if family else None def _get_search_context(results_model): diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 4e15831650..4ad0add591 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -12,7 +12,7 @@ get_audit_field_names, RnaSeqOutlier, RnaSeqSpliceOutlier, VariantSearchResults from seqr.utils.xpos_utils import get_chrom_pos from seqr.views.utils.json_utils import _to_camel_case -from seqr.views.utils.permissions_utils import has_project_permissions, \ +from seqr.views.utils.permissions_utils import has_project_edit_permission, \ project_has_anvil, get_workspace_collaborator_perms, user_is_analyst, user_is_data_manager, user_is_pm, \ is_internal_anvil_project, get_anvil_analyst_user_emails from seqr.views.utils.terra_api_utils import is_anvil_authenticated, anvil_enabled @@ -154,7 +154,7 @@ def _process_result(result, project): 'isAnalystProject': is_internal_anvil_project(project), }) if add_permissions: - result['canEdit'] = has_project_permissions(project, user, can_edit=True) + result['canEdit'] = has_project_edit_permission(project, user) if add_project_category_guids_field: result['projectCategoryGuids'] = list(project.projectcategory_set.values_list('guid', flat=True)) diff --git a/seqr/views/utils/permissions_utils.py b/seqr/views/utils/permissions_utils.py index d5a03f97cd..b3e80a4d26 100644 --- a/seqr/views/utils/permissions_utils.py +++ b/seqr/views/utils/permissions_utils.py @@ -110,9 +110,11 @@ def get_internal_projects(): return Project.objects.filter(workspace_namespace__in=INTERNAL_NAMESPACES) return Project.objects.all() +def get_project_and_check_edit_permission(project_guid, user): + return _get_project_and_check_permissions(project_guid, user, check_project_edit_permission) -def get_project_and_check_permissions(project_guid, user, **kwargs): - return _get_project_and_check_permissions(project_guid, user, check_project_permissions, **kwargs) +def get_project_and_check_view_permission(project_guid, user): + return _get_project_and_check_permissions(project_guid, user, _check_project_view_permission) def get_project_and_check_pm_permissions(project_guid, user, override_permission_func=None): return _get_project_and_check_permissions(project_guid, user, check_project_pm_permission, @@ -124,7 +126,7 @@ def _get_project_and_check_permissions(project_guid, user, _check_permission_fun return project def check_project_pm_permission(project, user, override_permission_func=None, **kwargs): - if user_is_pm(user) or (project.has_case_review and has_project_permissions(project, user, can_edit=True)): + if user_is_pm(user) or (project.has_case_review and has_project_edit_permission(project, user)): return if override_permission_func and override_permission_func(project, user): @@ -138,7 +140,7 @@ def project_has_anvil(project): def external_anvil_project_can_edit(project, user): - return project_has_anvil(project) and has_project_permissions(project, user, can_edit=True) and not \ + return project_has_anvil(project) and has_project_edit_permission(project, user) and not \ is_internal_anvil_project(project) @@ -186,14 +188,19 @@ def get_workspace_collaborator_perms(user, workspace_namespace, workspace_name): return permission_levels -def has_project_permissions(project, user, can_edit=False): - permission_level = CAN_VIEW - if can_edit: - permission_level = CAN_EDIT +def has_project_edit_permission(project, user): + return _has_project_permissions(project, user, CAN_EDIT) - return user_is_data_manager(user) or \ - (not can_edit and project.all_user_demo and project.is_demo) or \ - _user_project_permission(user, permission_level, project) +def _has_project_view_permission(project, user): + if project.all_user_demo and project.is_demo: + return True + return _has_project_permissions(project, user, CAN_VIEW) + +def has_family_view_permission(family, user): + return _has_project_view_permission(family.project, user) + +def _has_project_permissions(project, user, permission_level): + return user_is_data_manager(user) or _user_project_permission(user, permission_level, project) def _user_project_permission(user, permission_level, project): @@ -202,14 +209,28 @@ def _user_project_permission(user, permission_level, project): return user.has_perm(permission_level, project) -def check_project_permissions(project, user, **kwargs): - if has_project_permissions(project, user, **kwargs): +def check_project_edit_permission(project, user): + if has_project_edit_permission(project, user): return raise PermissionDenied("{user} does not have sufficient permissions for {project}".format( user=user, project=project)) +def _check_project_view_permission(project, user): + if _has_project_view_permission(project, user): + return + + raise PermissionDenied(f'{user} does not have sufficient permissions for {project}') + + +def check_family_view_permission(family, user): + if has_family_view_permission(family, user): + return + + raise PermissionDenied(f'{user} does not have sufficient permissions for {family}') + + def _is_user_created_object(obj, user): return obj.created_by == user @@ -224,8 +245,8 @@ def _get_all_can_view_project_guids_set(user): return set(get_project_guids_user_can_view(user, limit_data_manager=False)) -def check_projects_view_permission(projects, user): - no_access_projects = set(projects.values_list('guid', flat=True)) - _get_all_can_view_project_guids_set(user) +def check_families_view_permission(families, user): + no_access_projects = set(families.values_list('project__guid', flat=True).distinct()) - _get_all_can_view_project_guids_set(user) if no_access_projects: raise PermissionDenied(f"{user} does not have sufficient permissions for {','.join(no_access_projects)}") @@ -268,13 +289,14 @@ def get_project_guids_user_can_view(user, limit_data_manager=True): def check_mme_permissions(submission, user): - project = submission.individual.family.project - check_project_permissions(project, user) + family = submission.individual.family + project = family.project + check_family_view_permission(family, user) if not (project.is_mme_enabled and not project.is_demo): raise PermissionDenied('Matchmaker is not enabled') - return project + return project.genome_version def has_case_review_permissions(project, user): if not project.has_case_review: return False - return has_project_permissions(project, user, can_edit=True) + return has_project_edit_permission(project, user) diff --git a/ui/package-lock.json b/ui/package-lock.json index cbbc6ace05..45f3f5e6c4 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -2537,15 +2537,6 @@ "node": ">= 6" } }, - "node_modules/@trysound/sax": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz", - "integrity": "sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==", - "dev": true, - "engines": { - "node": ">=10.13.0" - } - }, "node_modules/@types/babel__core": { "version": "7.1.16", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.16.tgz", @@ -4525,9 +4516,9 @@ } }, "node_modules/babel-plugin-module-resolver/node_modules/minimatch": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.2.tgz", - "integrity": "sha512-bNH9mmM9qsJ2X4r2Nat1B//1dJVcn3+iBLa3IgqJ7EbGaDNepL9QSHOxN4ng33s52VMMhhIfgCYDk3C4ZmlDAg==", + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", "dev": true, "dependencies": { "brace-expansion": "^2.0.1" @@ -8551,15 +8542,15 @@ } }, "node_modules/flatted": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.1.1.tgz", - "integrity": "sha512-zAoAQiudy+r5SvnSw3KJy5os/oRJYHzrzja/tBDqrZtNhUw8bt6y8OBzMWcjWr+8liV8Eb6yOhw8WZ7VFZ5ZzA==", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", "dev": true }, "node_modules/follow-redirects": { - "version": "1.15.9", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", - "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", "dev": true, "funding": [ { @@ -8747,15 +8738,15 @@ } }, "node_modules/form-data": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.4.tgz", - "integrity": "sha512-f0cRzm6dkyVYV3nPoooP8XlccPQukegwhAnpoLcXy+X+A8KfpGOoXwDr9FLZd3wzgLaBGQBE3lY93Zm/i1JvIQ==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.5.tgz", + "integrity": "sha512-j23EibVLnp4zNXGW7LjryXYa2X6U/M96yoOX+ybZxwkYajdxRNEqYY3zhh7y0i6kfISKS2jr+EJq1YTUDEv5+w==", "dev": true, "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", + "hasown": "^2.0.4", "mime-types": "^2.1.35" }, "engines": { @@ -9189,9 +9180,9 @@ } }, "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", "dependencies": { "function-bind": "^1.1.2" }, @@ -12080,13 +12071,13 @@ } }, "node_modules/launch-editor": { - "version": "2.9.1", - "resolved": "https://registry.npmjs.org/launch-editor/-/launch-editor-2.9.1.tgz", - "integrity": "sha512-Gcnl4Bd+hRO9P9icCP/RVVT2o8SFlPXofuCxvA2SaZuH45whSvf5p8x5oih5ftLiVhEI4sp5xDY+R+b3zJBh5w==", + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/launch-editor/-/launch-editor-2.14.1.tgz", + "integrity": "sha512-QWBrQsMpH7gPr965dsKD/3cKWiNoTjpATQf++Xq63N6sKRGMwlVXz41O1IZTMfZQgBctD/K5Zt06+/I6pP6+HA==", "dev": true, "dependencies": { - "picocolors": "^1.0.0", - "shell-quote": "^1.8.1" + "picocolors": "^1.1.1", + "shell-quote": "^1.8.4" } }, "node_modules/leven": { @@ -12910,9 +12901,9 @@ "dev": true }, "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "dependencies": { "brace-expansion": "^1.1.7" @@ -15836,6 +15827,15 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "dev": true }, + "node_modules/sax": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.5.0.tgz", + "integrity": "sha512-21IYA3Q5cQf089Z6tgaUTr7lDAyzoTPx5HRtbhsME8Udispad8dC/+sziTNugOEx54ilvatQ9YCzl4KQLPcRHA==", + "dev": true, + "engines": { + "node": ">=11.0.0" + } + }, "node_modules/saxes": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", @@ -17112,17 +17112,17 @@ "dev": true }, "node_modules/svgo": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/svgo/-/svgo-2.7.0.tgz", - "integrity": "sha512-aDLsGkre4fTDCWvolyW+fs8ZJFABpzLXbtdK1y71CKnHzAnpDxKXPj2mNKj+pyOXUCzFHzuxRJ94XOFygOWV3w==", + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/svgo/-/svgo-2.8.2.tgz", + "integrity": "sha512-TyzE4NVGLUFy+H/Uy4N6c3G0HEeprsVfge6Lmq+0FdQQ/zqoVYB62IsBZORsiL+o96s6ff/V6/3UQo/C0cgCAA==", "dev": true, "dependencies": { - "@trysound/sax": "0.2.0", "commander": "^7.2.0", "css-select": "^4.1.3", "css-tree": "^1.1.3", "csso": "^4.2.0", - "nanocolors": "^0.1.12", + "picocolors": "^1.0.0", + "sax": "^1.5.0", "stable": "^0.1.8" }, "bin": { @@ -17141,12 +17141,6 @@ "node": ">= 10" } }, - "node_modules/svgo/node_modules/nanocolors": { - "version": "0.1.12", - "resolved": "https://registry.npmjs.org/nanocolors/-/nanocolors-0.1.12.tgz", - "integrity": "sha512-2nMHqg1x5PU+unxX7PGY7AuYxl2qDx7PSrTRjizr8sxdd3l/3hBuWWaki62qmtYm2U5i4Z5E7GbjlyDFhs9/EQ==", - "dev": true - }, "node_modules/symbol-tree": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", @@ -20988,12 +20982,6 @@ "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", "dev": true }, - "@trysound/sax": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz", - "integrity": "sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==", - "dev": true - }, "@types/babel__core": { "version": "7.1.16", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.16.tgz", @@ -22631,9 +22619,9 @@ } }, "minimatch": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.2.tgz", - "integrity": "sha512-bNH9mmM9qsJ2X4r2Nat1B//1dJVcn3+iBLa3IgqJ7EbGaDNepL9QSHOxN4ng33s52VMMhhIfgCYDk3C4ZmlDAg==", + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", "dev": true, "requires": { "brace-expansion": "^2.0.1" @@ -25930,15 +25918,15 @@ } }, "flatted": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.1.1.tgz", - "integrity": "sha512-zAoAQiudy+r5SvnSw3KJy5os/oRJYHzrzja/tBDqrZtNhUw8bt6y8OBzMWcjWr+8liV8Eb6yOhw8WZ7VFZ5ZzA==", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", "dev": true }, "follow-redirects": { - "version": "1.15.9", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", - "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", "dev": true }, "foreach": { @@ -26058,15 +26046,15 @@ } }, "form-data": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.4.tgz", - "integrity": "sha512-f0cRzm6dkyVYV3nPoooP8XlccPQukegwhAnpoLcXy+X+A8KfpGOoXwDr9FLZd3wzgLaBGQBE3lY93Zm/i1JvIQ==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.5.tgz", + "integrity": "sha512-j23EibVLnp4zNXGW7LjryXYa2X6U/M96yoOX+ybZxwkYajdxRNEqYY3zhh7y0i6kfISKS2jr+EJq1YTUDEv5+w==", "dev": true, "requires": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", + "hasown": "^2.0.4", "mime-types": "^2.1.35" } }, @@ -26387,9 +26375,9 @@ } }, "hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", "requires": { "function-bind": "^1.1.2" } @@ -28724,13 +28712,13 @@ } }, "launch-editor": { - "version": "2.9.1", - "resolved": "https://registry.npmjs.org/launch-editor/-/launch-editor-2.9.1.tgz", - "integrity": "sha512-Gcnl4Bd+hRO9P9icCP/RVVT2o8SFlPXofuCxvA2SaZuH45whSvf5p8x5oih5ftLiVhEI4sp5xDY+R+b3zJBh5w==", + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/launch-editor/-/launch-editor-2.14.1.tgz", + "integrity": "sha512-QWBrQsMpH7gPr965dsKD/3cKWiNoTjpATQf++Xq63N6sKRGMwlVXz41O1IZTMfZQgBctD/K5Zt06+/I6pP6+HA==", "dev": true, "requires": { - "picocolors": "^1.0.0", - "shell-quote": "^1.8.1" + "picocolors": "^1.1.1", + "shell-quote": "^1.8.4" } }, "leven": { @@ -29437,9 +29425,9 @@ "dev": true }, "minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "requires": { "brace-expansion": "^1.1.7" @@ -31752,6 +31740,12 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "dev": true }, + "sax": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.5.0.tgz", + "integrity": "sha512-21IYA3Q5cQf089Z6tgaUTr7lDAyzoTPx5HRtbhsME8Udispad8dC/+sziTNugOEx54ilvatQ9YCzl4KQLPcRHA==", + "dev": true + }, "saxes": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", @@ -32817,17 +32811,17 @@ "dev": true }, "svgo": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/svgo/-/svgo-2.7.0.tgz", - "integrity": "sha512-aDLsGkre4fTDCWvolyW+fs8ZJFABpzLXbtdK1y71CKnHzAnpDxKXPj2mNKj+pyOXUCzFHzuxRJ94XOFygOWV3w==", + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/svgo/-/svgo-2.8.2.tgz", + "integrity": "sha512-TyzE4NVGLUFy+H/Uy4N6c3G0HEeprsVfge6Lmq+0FdQQ/zqoVYB62IsBZORsiL+o96s6ff/V6/3UQo/C0cgCAA==", "dev": true, "requires": { - "@trysound/sax": "0.2.0", "commander": "^7.2.0", "css-select": "^4.1.3", "css-tree": "^1.1.3", "csso": "^4.2.0", - "nanocolors": "^0.1.12", + "picocolors": "^1.0.0", + "sax": "^1.5.0", "stable": "^0.1.8" }, "dependencies": { @@ -32836,12 +32830,6 @@ "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", "dev": true - }, - "nanocolors": { - "version": "0.1.12", - "resolved": "https://registry.npmjs.org/nanocolors/-/nanocolors-0.1.12.tgz", - "integrity": "sha512-2nMHqg1x5PU+unxX7PGY7AuYxl2qDx7PSrTRjizr8sxdd3l/3hBuWWaki62qmtYm2U5i4Z5E7GbjlyDFhs9/EQ==", - "dev": true } } }, diff --git a/ui/pages/DataManagement/components/LoadData.jsx b/ui/pages/DataManagement/components/LoadData.jsx index 5cbb9ead40..36e7664fdc 100644 --- a/ui/pages/DataManagement/components/LoadData.jsx +++ b/ui/pages/DataManagement/components/LoadData.jsx @@ -6,7 +6,7 @@ import { FormSpy } from 'react-final-form' import { getUser } from 'redux/selectors' import { validators } from 'shared/components/form/FormHelpers' import FormWizard from 'shared/components/form/FormWizard' -import { ButtonRadioGroup, InlineToggle } from 'shared/components/form/Inputs' +import { ButtonRadioGroup, CheckboxGroup, InlineToggle } from 'shared/components/form/Inputs' import LoadOptionsSelect from 'shared/components/form/LoadOptionsSelect' import { SAMPLE_TYPE_EXOME, @@ -55,17 +55,49 @@ const FILE_PATH_FIELD = { const CALLSET_PAGE_FIELDS = [ { - ...GENOME_VERSION_FIELD, + name: 'sampleType', + label: 'Sample Type', component: ButtonRadioGroup, + options: [SAMPLE_TYPE_EXOME, SAMPLE_TYPE_GENOME].map(value => ({ value, text: value })), validate: validators.required, }, { - name: 'sampleType', - label: 'Sample Type', + ...GENOME_VERSION_FIELD, component: ButtonRadioGroup, - options: [SAMPLE_TYPE_EXOME, SAMPLE_TYPE_GENOME].map(value => ({ value, text: value })), validate: validators.required, }, + { + name: 'validationsToSkip', + label: 'Skip Validations', + component: CheckboxGroup, + options: [ + { + value: 'validate_expected_contig_frequency', + text: 'Chromosome Frequency', + description: `By default, VCFs will be checked to ensure they have a reasonable number of variants in each + chromosome, and loading will fail if some chromosomes are missing data. If there is a known reason why some + chromosomes may be missing variants, such as with gene panel data, this validation may be safely skipped.`, + }, + { + value: 'validate_sample_type', + text: 'Sample Type', + description: `By default, VCFs will be checked for a representative sample of coding and non-coding SNPs which + will then be used to assess whether the selected Sample Type aligns with the provided data. If there is a + known reason why this validation would fail, such as WES data that uses broader capture regions, this + validation may be safely skipped.`, + }, + { + value: 'validate_no_duplicate_variants', + text: 'Duplicate Variants (modifies data before loading)', + description: `By default, VCFs will be checked to ensure they have no duplicate variants, as all supported + calling pipelines should have a single row per variant. While there are no well supported reasons why a VCF + with duplicate data should be loaded to seqr, we do provide this option to skip that validation and instead + deduplicate the data before running. NOTE: By selecting this option, variants will be ARBITRARILY DEDUPLICATED. + This means duplicate rows will be dropped at random to leave only one row per variant. If you want a more + deterministic approach to merging/ deduplicating your data, this should be done outside of seqr.`, + }, + ], + }, ] const CALLSET_PAGE = { @@ -91,13 +123,6 @@ const MULTI_DATA_TYPE_CALLSET_PAGE = { placeholder: 'gs://', ...FILE_PATH_FIELD, }, - { - name: 'skipSRChecks', - label: 'Skip Sex and Relatedness Checks', - component: InlineToggle, - asFormInput: true, - }, - ...CALLSET_PAGE_FIELDS, { name: 'datasetType', label: 'Dataset Type', @@ -109,6 +134,19 @@ const MULTI_DATA_TYPE_CALLSET_PAGE = { ].map(value => ({ value, text: value.replace('_', '/') })), validate: validators.required, }, + ...CALLSET_PAGE_FIELDS, + { + name: 'skipSRChecks', + label: 'Skip Sex and Relatedness Checks', + component: InlineToggle, + asFormInput: true, + }, + { + name: 'skipTDR', + label: 'Skip TDR Metrics', + component: InlineToggle, + asFormInput: true, + }, ], } diff --git a/ui/pages/Report/components/SeqrStats.jsx b/ui/pages/Report/components/SeqrStats.jsx index 26dfb7e6bd..ae030eae88 100644 --- a/ui/pages/Report/components/SeqrStats.jsx +++ b/ui/pages/Report/components/SeqrStats.jsx @@ -1,9 +1,10 @@ import React from 'react' import { connect } from 'react-redux' import PropTypes from 'prop-types' -import { Header, Table } from 'semantic-ui-react' +import { Header, Table, Grid } from 'semantic-ui-react' import { getUser } from 'redux/selectors' +import ExportTableButton from 'shared/components/buttons/ExportTableButton' import DataLoader from 'shared/components/DataLoader' import { DATASET_TITLE_LOOKUP } from 'shared/utils/constants' import { getSeqrStatsLoading, getSeqrStatsLoadingError, getSeqrStats } from '../selectors' @@ -22,38 +23,72 @@ const COLUMN_MAP = { DEMO_COLUMN, ], } +const USER_ROWS = [ + { title: 'Total', key: 'total' }, + { title: 'Logged In More Than Once', key: 'multipleLogins' }, + { title: 'Logged In Within 30 Days', key: 'lastMonth' }, + { title: 'Logged In Within 365 Days', key: 'lastYear' }, + { title: 'Logged In This Year', key: 'thisYear' }, +] + +const EXPORT_SAMPLES_CONFIG = [{ name: 'Sample History', url: '/api/report/sample_stats_download' }] const SeqrStats = React.memo(({ stats, error, loading, load, user }) => (
- - - - - {COLUMN_MAP[user.isAnvil].map(({ title }) => )} - - - {['Projects', 'Families', 'Individuals'].map(field => ( - - - {COLUMN_MAP[user.isAnvil].map(({ key }) => ( - - ))} - - ))} - {Object.keys(stats.sampleCountsByType || {}).sort().map(sampleTypes => ( - - - {COLUMN_MAP[user.isAnvil].map(({ key }) => ( - - ))} - - ))} -
+ + + + + + + + {COLUMN_MAP[user.isAnvil].map(({ title }) => )} + + + {['Projects', 'Families', 'Individuals'].map(field => ( + + + {COLUMN_MAP[user.isAnvil].map(({ key }) => ( + + ))} + + ))} + {Object.keys(stats.sampleCountsByType || {}).sort().map(sampleTypes => ( + + + {COLUMN_MAP[user.isAnvil].map(({ key }) => ( + + ))} + + ))} +
+ + + + + + + + + + + + {USER_ROWS.map(({ title, key }) => ( + + + + + ))} + +
+ + +
)) diff --git a/ui/shared/components/form/Inputs.jsx b/ui/shared/components/form/Inputs.jsx index ed4bd2f797..e374e6c206 100644 --- a/ui/shared/components/form/Inputs.jsx +++ b/ui/shared/components/form/Inputs.jsx @@ -315,7 +315,7 @@ const InlineFormGroup = styled(Form.Group).attrs({ inline: true })` ` const selectAll = (onChange, value, options) => ({ checked }) => { - const remainValue = value.filter(val => !options.find(opt => opt.value === val)) + const remainValue = (value || []).filter(val => !options.find(opt => opt.value === val)) if (checked) { onChange(options.map(option => option.value).concat(remainValue)) } else { @@ -325,9 +325,9 @@ const selectAll = (onChange, value, options) => ({ checked }) => { const selectCheckbox = (onChange, value, option) => ({ checked }) => { if (checked) { - onChange([...value, option.value]) + onChange([...(value || []), option.value]) } else { - onChange(value.filter(val => val !== option.value)) + onChange((value || []).filter(val => val !== option.value)) } } @@ -340,7 +340,7 @@ const chunkArray = (arr, maxChunkSize) => { export const CheckboxGroup = React.memo((props) => { const { value, label, groupLabel, onChange, maxOptionsPerColumn, inline, ...baseProps } = props const options = props.options.map(styledOption) - const numSelected = options.filter(opt => value.includes(opt.value)).length + const numSelected = options.filter(opt => value?.includes(opt.value)).length const optionGroups = maxOptionsPerColumn && options.length > maxOptionsPerColumn ? chunkArray(options, maxOptionsPerColumn) : [options] const optionLists = optionGroups.map(optionGroup => ( @@ -350,7 +350,7 @@ export const CheckboxGroup = React.memo((props) => {