Dev

clickhouse_search/all_search_tests.py

@@ -2408,7 +2408,7 @@ def test_gene_variant_lookup(self):
             cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}],
             annotations=annotations, freqs=freqs, locus=locus, project_families=[], export_data=[
                 EXPORT_DATA[0][:27], EXPORT_DATA[4][:24] + ['3 Families', '', ''],
-            ],
+            ], gene_counts={'ENSG00000097046': {'total': 1, 'families': {}}},
         )
 
         freqs = {'callset': freqs['callset']}
@@ -2443,6 +2443,10 @@ def test_gene_variant_lookup(self):
                 {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]},
             ],
             annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', project_families=[],
+            gene_counts={
+                'ENSG00000097046': {'total': 2, 'families': {}},
+                'ENSG00000177000': {'total': 1, 'families': {}},
+            },
         )
 
         self.login_collaborator()
@@ -2455,7 +2459,10 @@ def test_gene_variant_lookup(self):
                 {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[3][3]},
                 {'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]},
             ],
-            annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo',
+            annotations=annotations, freqs=freqs, locus=locus, inheritance_mode='de_novo', gene_counts={
+                'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}},
+                'ENSG00000177000': {'total': 1, 'families': {'F000003_3': 1}},
+            },
         )
 
         locus['rawItems'] = 'ENSG00000171621'

clickhouse_search/fixtures/variant_details_for_update.json

@@ -83,7 +83,7 @@
         "key": 6,
         "variant_id": "1-94818-T-C",
         "rsid": null,
-        "caid": null,
+        "caid": "CA16717153",
         "lifted_over_chrom": "1",
         "lifted_over_pos": 94818,
         "sorted_regulatory_feature_consequences": [],

clickhouse_search/management/commands/register_caids.py

@@ -224,7 +224,7 @@ def handle_api_response(
 def register_caids(
     genome_version: Literal[GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38],
     variants: Union[list[VariantDetailsGRCh37SnvIndel], list[VariantDetailsSnvIndel]],
-) -> int:
+) -> tuple[int, Union[list[VariantDetailsGRCh37SnvIndel], list[VariantDetailsSnvIndel]]]:
     rows = list(ALLELE_REGISTRY_HEADERS[genome_version]) # NB: new copy of the list
     for variant in variants:
         chrom, pos, ref, alt = variant.variant_id.split("-")
@@ -254,10 +254,14 @@ def register_caids(
     )
     mapped_variants = handle_api_response(genome_version, res)
     max_key_id = -1
+    update_variants = []
     for variant in variants:
-        variant.caid = mapped_variants.get(variant.variant_id, None)
+        caid = mapped_variants.get(variant.variant_id)
+        if caid:
+            variant.caid = caid
+            update_variants.append(variant)
         max_key_id = max(max_key_id, variant.key_id)
-    return max_key_id
+    return max_key_id, update_variants
 
 def join_series(model: Union[VariantDetailsSnvIndel, VariantDetailsGRCh37SnvIndel], min_: int, max_: int):
     table = model._meta.db_table
@@ -316,8 +320,8 @@ def handle(self, *args, **options):
                     break
 
                 try:
-                    max_key = register_caids(genome_version, variants)
-                    variant_details_model.objects.using('clickhouse_write').bulk_update(variants, ["caid"])
+                    max_key, update_variants = register_caids(genome_version, variants)
+                    variant_details_model.objects.using('clickhouse_write').bulk_update(update_variants, ["caid"])
                     # Save current key on every iteration
                     curr_key = max_key
                     version_obj.version = curr_key

clickhouse_search/management/tests/register_caids_tests.py

@@ -269,8 +269,16 @@ def test_register_caids(self, mock_safe_post_to_slack, mock_logger):
             'MESSAGE: 1\t10469\trs370233998\tC\tG\t.\t.\t.\n'
             'INPUT_LINE: Cannot align NC_000001.10 [10468,10469).',
         )
-        vd = VariantDetailsSnvIndel.objects.get(variant_id='1-91511686-T-G')
-        self.assertEqual(vd.caid, 'CA997563840')
+        self.assertListEqual(list(VariantDetailsSnvIndel.objects.order_by('key').values_list('variant_id', 'caid')), [
+            ('1-10439-AC-A', 'CA16717152'),
+            ('1-38724419-T-G', None),
+            ('1-91502721-G-A', None),
+            ('1-91511686-T-G', 'CA997563840'),
+            ('1-10146-ACC-A', 'CA997563845'),
+            ('1-94818-T-C', 'CA16717153'),
+            ('7-143270172-A-G', None),
+            ('7-9310123-T-C', None)
+        ])
 
         # Ensure re-calling is a no-op
         responses.reset()

matchmaker/views/matchmaker_api.py

@@ -20,8 +20,8 @@
 from seqr.views.utils.json_utils import create_json_response
 from seqr.views.utils.orm_to_json_utils import _get_json_for_model, get_json_for_saved_variants, \
     get_json_for_matchmaker_submission, get_json_for_matchmaker_submissions
-from seqr.views.utils.permissions_utils import check_mme_permissions, check_project_permissions, analyst_required, \
-    has_project_permissions, login_and_policies_required, get_project_and_check_permissions
+from seqr.views.utils.permissions_utils import check_mme_permissions, check_family_view_permission, analyst_required, \
+    has_family_view_permission, login_and_policies_required, get_project_and_check_edit_permission
 
 from settings import BASE_URL, MME_ACCEPT_HEADER, MME_NODES, MME_DEFAULT_CONTACT_EMAIL, \
     MME_SLACK_SEQR_MATCH_NOTIFICATION_CHANNEL, MME_SLACK_ALERT_NOTIFICATION_CHANNEL, VLM_SEND_EMAIL
@@ -36,7 +36,7 @@
 @login_and_policies_required
 def get_individual_mme_matches(request, submission_guid):
     submission = MatchmakerSubmission.objects.get(guid=submission_guid)
-    project = check_mme_permissions(submission, request.user)
+    genome_version = check_mme_permissions(submission, request.user)
 
     results = MatchmakerResult.objects.filter(submission=submission)
 
@@ -51,7 +51,7 @@ def get_individual_mme_matches(request, submission_guid):
 
     variants = get_json_for_saved_variants(
         SavedVariant.objects.filter(guid__in=variant_guids), additional_values={
-            'genomeVersion': Value(project.genome_version),
+            'genomeVersion': Value(genome_version),
             'selectedMainTranscript': F('main_transcript'),
             'xposEnd': F('xpos_end'),
         },
@@ -273,7 +273,7 @@ def update_mme_submission(request, submission_guid=None):
         if not individual_guid:
             return create_json_response({}, status=400, reason='Individual is required for a new submission')
         individual = Individual.objects.get(guid=individual_guid)
-        check_project_permissions(individual.family.project, request.user)
+        check_family_view_permission(individual.family, request.user)
         submission = create_model_from_json(MatchmakerSubmission, {
             'individual': individual,
             'submission_id': individual.guid,
@@ -390,7 +390,7 @@ def send_mme_contact_email(request, matchmaker_result_guid):
 
 @login_and_policies_required
 def update_mme_project_contact(request, project_guid):
-    project = get_project_and_check_permissions(project_guid, request.user, can_edit=True)
+    project = get_project_and_check_edit_permission(project_guid, request.user)
 
     request_json = json.loads(request.body)
     contact = (request_json.get('contact') or '').strip()
@@ -434,7 +434,7 @@ def _parse_mme_results(submission, saved_results, user, additional_genes=None, r
         result['matchStatus'] = _get_json_for_model(result_model)
         if result_model.originating_submission:
             originating_family = result_model.originating_submission.individual.family
-            if has_project_permissions(originating_family.project, user):
+            if has_family_view_permission(originating_family, user):
                 result['originatingSubmission'] = {
                     'originatingSubmissionGuid': result_model.originating_submission.guid,
                     'familyGuid': originating_family.guid,

seqr/urls.py

@@ -125,6 +125,7 @@
     family_metadata, \
     variant_metadata, \
     gregor_export, \
+    sample_stats_download, \
     seqr_stats
 from seqr.views.apis.summary_data_api import success_story, saved_variants_page, mme_details, hpo_summary_data, \
     bulk_update_family_external_analysis, individual_metadata
@@ -325,6 +326,7 @@
     'report/variant_metadata/(?P<project_guid>[^/]+)': variant_metadata,
     'report/gregor': gregor_export,
     'report/seqr_stats': seqr_stats,
+    'report/sample_stats_download': sample_stats_download,
 
     'data_management/get_all_users': get_all_users,
     'data_management/update_rna_seq': update_rna_seq,

seqr/utils/add_data_utils.py

@@ -90,7 +90,7 @@ def trigger_rebuild_gt_stats(project, user):
 
 def trigger_data_loading(projects: list[Project], individual_ids: list[int], sample_type: str, dataset_type: str,
                          genome_version: str, data_path: str, user: User, raise_error: bool = False, skip_expect_tdr_metrics: bool = True,
-                         skip_check_sex_and_relatedness: bool = True, vcf_sample_id_map=None,
+                         skip_check_sex_and_relatedness: bool = True, validations_to_skip: list[str] = None, vcf_sample_id_map=None,
                          success_message: str = None,  error_message: str = None, success_slack_channel: str = SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL):
     variables = {
         'projects_to_run': sorted([p.guid for p in projects]) if projects else None,
@@ -99,11 +99,12 @@ def trigger_data_loading(projects: list[Project], individual_ids: list[int], sam
         'callset_path': data_path,
         'sample_type': sample_type,
     }
-    bool_variables = {
+    conditional_variables = {
         'skip_check_sex_and_relatedness': skip_check_sex_and_relatedness,
         'skip_expect_tdr_metrics': skip_expect_tdr_metrics,
+        'validations_to_skip': _valid_validation_to_skip(validations_to_skip, user),
     }
-    variables.update({k: v for k, v in bool_variables.items() if v})
+    variables.update({k: v for k, v in conditional_variables.items() if v})
     file_path = _get_pedigree_path(genome_version, sample_type, dataset_type)
     _upload_data_loading_files(individual_ids, vcf_sample_id_map or {}, user, file_path, raise_error)
     _write_gene_id_file(user)
@@ -126,6 +127,16 @@ def trigger_data_loading(projects: list[Project], individual_ids: list[int], sam
     return success
 
 
+def _valid_validation_to_skip(validations_to_skip: list[str], user: User) -> list[str]:
+    invalid_validations = [validation for validation in (validations_to_skip or []) if validation not in {
+        'validate_expected_contig_frequency', 'validate_sample_type', 'validate_no_duplicate_variants',
+    }]
+    if invalid_validations:
+        validations_to_skip = [validation for validation in validations_to_skip if validation not in invalid_validations]
+        logger.error(f'Omitted invalid skip validations from loading request: {", ".join(invalid_validations)}', user)
+    return validations_to_skip
+
+
 def _enqueue_pipeline_request(name: str, variables: dict, user: User, raise_error: bool = True, log_error: bool = True):
     response = None
     error = None
@@ -185,7 +196,7 @@ def _upload_data_loading_files(individual_ids: list[int], vcf_sample_id_map: dic
 
 def _write_gene_id_file(user):
     file_name = 'db_id_to_gene_id'
-    if does_file_exist(f'{LOADING_DATASETS_DIR}/{file_name}.csv.gz'):
+    if does_file_exist(f'{LOADING_DATASETS_DIR}/{file_name}.csv.gz', user):
         return
 
     gene_data_loaded = (GeneInfo.objects.filter(gencode_release=int(GeneInfo.CURRENT_VERSION)).exists() and

seqr/views/apis/analysis_group_api.py

@@ -4,15 +4,15 @@
 from seqr.views.utils.json_utils import create_json_response
 from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json
 from seqr.views.utils.orm_to_json_utils import get_json_for_analysis_group
-from seqr.views.utils.permissions_utils import get_project_and_check_permissions, login_and_policies_required
+from seqr.views.utils.permissions_utils import get_project_and_check_edit_permission, login_and_policies_required
 
 
 REQUIRED_FIELDS = {'name': 'Name', 'familyGuids': 'Families'}
 
 
 def _update_analysis_group(request, project_guid, analysis_group_guid, model_cls, required_fields, is_dynamic=False,
                            validate_body=lambda x: None, post_process_model=lambda x: None):
-    project = get_project_and_check_permissions(project_guid, request.user, can_edit=True)
+    project = get_project_and_check_edit_permission(project_guid, request.user)
 
     request_json = json.loads(request.body)
     missing_fields = [field for field in required_fields.keys() if not request_json.get(field)]
@@ -79,7 +79,7 @@ def update_dynamic_analysis_group_handler(request, project_guid, analysis_group_
 
 @login_and_policies_required
 def delete_analysis_group_handler(request, project_guid, analysis_group_guid, model_cls=AnalysisGroup):
-    project = get_project_and_check_permissions(project_guid, request.user, can_edit=True)
+    project = get_project_and_check_edit_permission(project_guid, request.user)
     model_cls.objects.get(guid=analysis_group_guid, project=project).delete_model(request.user, user_can_delete=True)
 
     return create_json_response({'analysisGroupsByGuid': {analysis_group_guid: None}})

seqr/views/apis/data_manager_api.py

@@ -304,19 +304,18 @@ def load_data(request):
     if errors:
         raise ErrorsWarningsException(errors)
 
-    is_local = True
-    success_message = None
-    error_message = None
-    if AirtableSession.is_airtable_enabled():
-        is_local = False
-        success_message = f'*{request.user.email}* triggered loading internal {sample_type} {dataset_type} data for {len(individual_ids)} samples in {len(projects)} projects ({"; ".join(sorted(project_counts))})'
-        error_message = f'ERROR triggering internal {sample_type} {dataset_type} loading'
+    loading_kwargs = {
+        'skip_check_sex_and_relatedness': request_json.get('skipSRChecks', False),
+        'skip_expect_tdr_metrics': request_json.get('skipTDR', False),
+        'vcf_sample_id_map': vcf_sample_id_map,
+        'success_message': f'*{request.user.email}* triggered loading internal {sample_type} {dataset_type} data for {len(individual_ids)} samples in {len(projects)} projects ({"; ".join(sorted(project_counts))})',
+        'error_message': f'ERROR triggering internal {sample_type} {dataset_type} loading',
+    } if AirtableSession.is_airtable_enabled() else {'raise_error': True}
 
     success = trigger_data_loading(
         projects_by_guid.values(), individual_ids, sample_type, dataset_type, request_json['genomeVersion'],
-        _callset_path(request_json), user=request.user,
-        skip_check_sex_and_relatedness=request_json.get('skipSRChecks', False), vcf_sample_id_map=vcf_sample_id_map,
-        raise_error=is_local, skip_expect_tdr_metrics=is_local, success_message=success_message, error_message=error_message,
+        _callset_path(request_json), user=request.user, validations_to_skip=request_json.get('validationsToSkip'),
+        **loading_kwargs,
     )
 
     return create_json_response({'success': success})