Dev

clickhouse_search/constants.py

@@ -79,6 +79,42 @@
     ('benign', 'Benign/Likely_benign', 'Benign'),
 ]
 
+CLINVAR_ASSERTIONS = [
+    'Affects',
+    'association',
+    'association_not_found',
+    'confers_sensitivity',
+    'drug_response',
+    'low_penetrance',
+    'not_provided',
+    'other',
+    'protective',
+    'risk_factor',
+    'no_classification_for_the_single_variant',
+    'no_classifications_from_unflagged_records',
+]
+CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY = 'Conflicting_classifications_of_pathogenicity'
+CLINVAR_DEFAULT_PATHOGENICITY = 'No_pathogenic_assertion'
+CLINVAR_PATHOGENICITIES = [
+    'Pathogenic',
+    'Pathogenic/Likely_pathogenic',
+    'Pathogenic/Likely_pathogenic/Established_risk_allele',
+    'Pathogenic/Likely_pathogenic/Likely_risk_allele',
+    'Pathogenic/Likely_risk_allele',
+    'Likely_pathogenic',
+    'Likely_pathogenic/Likely_risk_allele',
+    'Established_risk_allele',
+    'Likely_risk_allele',
+    CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY,
+    'Uncertain_risk_allele',
+    'Uncertain_significance/Uncertain_risk_allele',
+    'Uncertain_significance',
+    CLINVAR_DEFAULT_PATHOGENICITY,
+    'Likely_benign',
+    'Benign/Likely_benign',
+    'Benign',
+]
+
 HGMD_KEY = 'hgmd'
 HGMD_CLASS_FILTERS = [
     ('disease_causing', 'DM'),

clickhouse_search/management/commands/reload_clinvar_all_variants.py

@@ -15,6 +15,9 @@
 from clickhouse_backend import models
 from clickhouse_search.models.reference_data_models import ClinvarAllVariantsGRCh37SnvIndel, ClinvarAllVariantsSnvIndel, ClinvarAllVariantsMito, \
     ClinvarMvGRCh37SnvIndel, ClinvarMvSnvIndel, ClinvarMvMito, ClinvarSearchMvGRCh37SnvIndel, ClinvarSearchMvSnvIndel, ClinvarSearchMvMito
+from clickhouse_search.constants import \
+    CLINVAR_ASSERTIONS as CORE_CLINVAR_ASSERTIONS, CLINVAR_PATHOGENICITIES as CORE_CLINVAR_PATHOGENICITIES, CLINVAR_DEFAULT_PATHOGENICITY as CORE_CLINVAR_DEFAULT_PATHOGENICITY, \
+    CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY as CORE_CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY
 from reference_data.models import DataVersions
 from seqr.utils.communication_utils import safe_post_to_slack
 
@@ -29,11 +32,11 @@ def replace_spaces_with_underscores(value: Union[list[str], list[tuple[str, int]
     return [s.replace(' ', '_') for s in value]
 
 BATCH_SIZE = 1000
-CLINVAR_ASSERTIONS = replace_underscores_with_spaces(ClinvarAllVariantsSnvIndel.CLINVAR_ASSERTIONS)
-CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY = replace_underscores_with_spaces([ClinvarAllVariantsSnvIndel.CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY])[0]
+CLINVAR_ASSERTIONS = replace_underscores_with_spaces(CORE_CLINVAR_ASSERTIONS)
+CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY = replace_underscores_with_spaces([CORE_CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY])[0]
 CLINVAR_CONFLICTING_DATA_FROM_SUBMITTERS = 'conflicting data from submitters'
-CLINVAR_DEFAULT_PATHOGENICITY = replace_underscores_with_spaces([ClinvarAllVariantsSnvIndel.CLINVAR_DEFAULT_PATHOGENICITY])[0]
-CLINVAR_PATHOGENICITIES = replace_underscores_with_spaces(ClinvarAllVariantsSnvIndel.CLINVAR_PATHOGENICITIES)
+CLINVAR_DEFAULT_PATHOGENICITY = replace_underscores_with_spaces([CORE_CLINVAR_DEFAULT_PATHOGENICITY])[0]
+CLINVAR_PATHOGENICITIES = replace_underscores_with_spaces(CORE_CLINVAR_PATHOGENICITIES)
 CLINVAR_GOLD_STARS_LOOKUP = {
     'no classification for the single variant': 0,
     'no classification provided': 0,

clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py

@@ -197,7 +197,7 @@ def test_batching(self, mock_logger, mock_safe_post_to_slack):
         call_command('reload_clinvar_all_variants')
         mock_logger.assert_called_with('Updating Clinvar ClickHouse tables to 2025-06-30 from 2025-06-23.')
         self.assertEqual(ClinvarAllVariantsSnvIndel.objects.all().count(), BATCH_SIZE * 2 + 10)
-        self.assertEqual(ClinvarAllVariantsSnvIndel.objects.first().pathogenicity, ClinvarAllVariantsSnvIndel.CLINVAR_DEFAULT_PATHOGENICITY)
+        self.assertEqual(ClinvarAllVariantsSnvIndel.objects.first().pathogenicity, 'No_pathogenic_assertion')
         self.assertIsNone(ClinvarAllVariantsSnvIndel.objects.first().gold_stars)
 
     @responses.activate
@@ -417,7 +417,7 @@ def test_conflicting_data_from_submitters(self, mock_logger, mock_safe_post_to_s
         )
         call_command('reload_clinvar_all_variants')
         self.assertEqual(ClinvarAllVariantsSnvIndel.objects.count(), 1)
-        self.assertEqual(ClinvarAllVariantsSnvIndel.objects.first().pathogenicity, ClinvarAllVariantsSnvIndel.CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY)
+        self.assertEqual(ClinvarAllVariantsSnvIndel.objects.first().pathogenicity, 'Conflicting_classifications_of_pathogenicity')
         mock_safe_post_to_slack.assert_called_with(
             SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL,
             'Successfully updated Clinvar ClickHouse tables to 2025-06-30.',

clickhouse_search/models/reference_data_models.py

@@ -6,6 +6,7 @@
 from clickhouse_search.backend.fields import Enum8Field, NestedField, UInt32FieldDeltaCodecField, DictKeyForeignKey
 from clickhouse_search.backend.table_models import FixtureLoadableClickhouseModel, Dictionary, \
     RefreshableMaterializedView, RefreshableMaterializedViewMeta
+from clickhouse_search.constants import CLINVAR_ASSERTIONS, CLINVAR_PATHOGENICITIES
 from seqr.utils.xpos_utils import CHROMOSOME_CHOICES
 from settings import DATABASES, PIPELINE_RUNNER_SERVER
 
@@ -39,43 +40,6 @@ class ReferenceDataDictMeta:
 
 
 class BaseClinvar(FixtureLoadableClickhouseModel):
-
-    CLINVAR_ASSERTIONS = [
-        'Affects',
-        'association',
-        'association_not_found',
-        'confers_sensitivity',
-        'drug_response',
-        'low_penetrance',
-        'not_provided',
-        'other',
-        'protective',
-        'risk_factor',
-        'no_classification_for_the_single_variant',
-        'no_classifications_from_unflagged_records',
-    ]
-    CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY = 'Conflicting_classifications_of_pathogenicity'
-    CLINVAR_DEFAULT_PATHOGENICITY = 'No_pathogenic_assertion'
-    CLINVAR_PATHOGENICITIES = [
-        'Pathogenic',
-        'Pathogenic/Likely_pathogenic',
-        'Pathogenic/Likely_pathogenic/Established_risk_allele',
-        'Pathogenic/Likely_pathogenic/Likely_risk_allele',
-        'Pathogenic/Likely_risk_allele',
-        'Likely_pathogenic',
-        'Likely_pathogenic/Likely_risk_allele',
-        'Established_risk_allele',
-        'Likely_risk_allele',
-        CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY,
-        'Uncertain_risk_allele',
-        'Uncertain_significance/Uncertain_risk_allele',
-        'Uncertain_significance',
-        CLINVAR_DEFAULT_PATHOGENICITY,
-        'Likely_benign',
-        'Benign/Likely_benign',
-        'Benign',
-    ]
-
     ASSERTIONS_CHOICES = list(enumerate(CLINVAR_ASSERTIONS))
     PATHOGENICITY_CHOICES = list(enumerate(CLINVAR_PATHOGENICITIES))
 

seqr/fixtures/1kg_project.json

@@ -757,7 +757,7 @@
         "sex": "M",
         "affected": "A",
         "display_name": "",
-	"notes": "",
+        "features": [{"id": "HP:0001508"}],
         "case_review_status": "",
         "case_review_status_last_modified_date": null,
         "case_review_status_last_modified_by": null,

seqr/utils/add_data_utils.py

@@ -63,7 +63,7 @@ def update_airtable_loading_tracking_status(project, status, additional_update=N
         update={'Status': status, **(additional_update or {})},
     )
 
-def trigger_delete_families_search(project, family_guids, user=None):
+def trigger_delete_families_search(project, family_guids, user=None, dataset_types=None):
     num_updated = 0
     updated_families = set()
     for dataset in Dataset.objects.filter(active_individuals__family__guid__in=family_guids).distinct():
@@ -80,6 +80,8 @@ def trigger_delete_families_search(project, family_guids, user=None):
         logger.info(message, user)
 
     variables = {'project_guid': project.guid, 'family_guids': family_guids}
+    if dataset_types:
+        variables['dataset_types'] = dataset_types
     _enqueue_pipeline_request('delete_families', variables, user)
     info.append('Triggered delete family data')
     return info

seqr/views/apis/anvil_workspace_api.py

@@ -253,10 +253,10 @@ def _validate_expected_samples(vcf_samples, loaded_sample_types, loaded_individu
 def _trigger_add_workspace_data(project, pedigree_records, user, data_path, sample_type, previous_loaded_ids=None, get_pedigree_json=False):
     # add families and individuals according to the uploaded individual records
     pedigree_json, individual_ids = add_or_update_individuals_and_families(
-        project, individual_records=pedigree_records, user=user, get_update_json=get_pedigree_json, get_updated_individual_db_ids=True,
+        project, individual_records=pedigree_records, user=user, get_update_json=get_pedigree_json, get_individual_db_ids=True,
         allow_features_update=True, skip_gt_stats_rebuild=True,
     )
-    num_updated_individuals = len(individual_ids)
+    num_updated_individuals = len(individual_ids - set(previous_loaded_ids or []))
     individual_ids.update(previous_loaded_ids or [])
 
     # use airflow api to trigger AnVIL dags

seqr/views/apis/anvil_workspace_api_tests.py

@@ -565,13 +565,10 @@ def setUp(self):
         patcher = mock.patch('seqr.utils.add_data_utils.logger')
         self.mock_add_data_utils_logger = patcher.start()
         self.addCleanup(patcher.stop)
-        patcher = mock.patch('seqr.views.apis.anvil_workspace_api.load_uploaded_file')
-        self.mock_load_file = patcher.start()
-        self.mock_load_file.return_value = LOAD_SAMPLE_DATA
-        self.addCleanup(patcher.stop)
         patcher = mock.patch('seqr.utils.file_utils.subprocess.Popen')
         self.mock_subprocess = patcher.start()
         self.mock_subprocess.return_value.wait.return_value = 0
+        self.mock_subprocess.return_value.stdout = [json.dumps(LOAD_SAMPLE_DATA).encode('utf-8')]
         self.addCleanup(patcher.stop)
         patcher = mock.patch('seqr.views.utils.export_utils.TemporaryDirectory')
         mock_tempdir = patcher.start()
@@ -596,6 +593,9 @@ def setUp(self):
 
         super().setUp()
 
+    def _set_load_file_iter(self, data):
+        self.mock_subprocess.return_value.stdout = [json.dumps(data).encode('utf-8')]
+
     @mock.patch('seqr.models.Family._compute_guid', lambda family: f'F_{family.family_id}_{family.project.workspace_name[17:]}')
     @mock.patch('seqr.models.Project._compute_guid', lambda project: f'P_{project.name}')
     @responses.activate
@@ -612,8 +612,8 @@ def test_create_project_from_workspace(self):
 
         # Test valid operation
         responses.calls.reset()
-        self.mock_load_file.return_value = LOAD_SAMPLE_DATA
-        self.mock_subprocess.return_value.wait.side_effect = [0, 1, 0]
+        self._set_load_file_iter(LOAD_SAMPLE_DATA)
+        self.mock_subprocess.return_value.wait.side_effect = [0, 0, 1, 0]
         response = self.client.post(url, content_type='application/json', data=json.dumps(REQUEST_BODY))
         self.assertEqual(response.status_code, 200)
         project = Project.objects.get(workspace_namespace=TEST_WORKSPACE_NAMESPACE, workspace_name=TEST_NO_PROJECT_WORKSPACE_NAME)
@@ -685,14 +685,14 @@ def test_add_workspace_data(self, mock_compute_indiv_guid):
         self._test_errors(url, ['uploadedFileId', 'fullDataPath', 'vcfSamples'], TEST_WORKSPACE_NAME, has_existing_data=True)
 
         # Test loading data from empty ped file
-        self.mock_load_file.return_value = LOAD_SAMPLE_DATA[:1]
+        self._set_load_file_iter(LOAD_SAMPLE_DATA[:1])
         response = self.client.post(url, content_type='application/json', data=json.dumps(REQUEST_BODY))
         self.assertEqual(response.status_code, 400)
         response_json = response.json()
         self.assertListEqual(response_json['errors'], ['No samples found in the pedigree file'])
 
         # Test Individual ID exists in an omitted family and missing loaded samples
-        self.mock_load_file.return_value = LOAD_SAMPLE_DATA + INVALID_ADDED_SAMPLE_DATA
+        self._set_load_file_iter(LOAD_SAMPLE_DATA + INVALID_ADDED_SAMPLE_DATA)
         response = self.client.post(url, content_type='application/json', data=json.dumps(REQUEST_BODY))
         self.assertEqual(response.status_code, 400)
         response_json = response.json()
@@ -705,7 +705,7 @@ def test_add_workspace_data(self, mock_compute_indiv_guid):
         ])
 
         # Test project still has pending loading
-        self.mock_load_file.return_value = LOAD_SAMPLE_DATA
+        self._set_load_file_iter(LOAD_SAMPLE_DATA)
         response = self.client.post(url, content_type='application/json', data=json.dumps(REQUEST_BODY_ADD_DATA))
         self.assertEqual(response.status_code, 400)
         self.assertListEqual(response.json()['errors'], [
@@ -714,8 +714,8 @@ def test_add_workspace_data(self, mock_compute_indiv_guid):
         ])
 
         # Test a valid operation
-        self.mock_subprocess.return_value.wait.side_effect = [0, 1, 0]
-        self.mock_load_file.return_value = LOAD_SAMPLE_DATA_ALL_PENDING
+        self.mock_subprocess.return_value.wait.side_effect = [0, 0, 1, 0]
+        self._set_load_file_iter(LOAD_SAMPLE_DATA_ALL_PENDING)
         mock_compute_indiv_guid.side_effect = ['I0000020_hg00735', 'I0000021_hg00736']
         response = self.client.post(url, content_type='application/json', data=json.dumps(REQUEST_BODY_ADD_DATA))
         self.assertEqual(response.status_code, 200)
@@ -724,17 +724,17 @@ def test_add_workspace_data(self, mock_compute_indiv_guid):
         self.assertSetEqual(set(response_json['individualsByGuid'].keys()), {
             'I0000020_hg00735', 'I000001_na19675', 'I000003_na19679', 'I0000021_hg00736', 'I000007_na20870',
             'I000009_na20874', 'I000004_hg00731', 'I000010_na20875', 'I000008_na20872', 'I000005_hg00732',
-            'I000006_hg00733', 'I000013_na20878', 'I000012_na20877',
+            'I000006_hg00733', 'I000012_na20877',
         })
         self.assertSetEqual(set(response_json['familiesByGuid'].keys()), {
-            'F000001_1', 'F000015_21', 'F000006_6', 'F000013_13', 'F000005_5', 'F000009_9', 'F000008_8', 'F000004_4',
+            'F000001_1', 'F000015_21', 'F000006_6', 'F000013_13', 'F000005_5', 'F000008_8', 'F000004_4',
             'F000002_2', 'F000003_3',
         })
         self.assertEqual(list(response_json['familyNotesByGuid'].keys()), ['FAN000005_21_c_a_new_family'])
 
         self._assert_valid_operation(Project.objects.get(guid=PROJECT1_GUID))
 
-        self.mock_load_file.return_value = LOAD_SAMPLE_DATA_ALL_PENDING_PROJECT_2
+        self._set_load_file_iter(LOAD_SAMPLE_DATA_ALL_PENDING_PROJECT_2)
         mock_compute_indiv_guid.side_effect = ['I0000021_na19675_1', 'I0000022_na19678', 'I0000023_hg00735']
         url = reverse(add_workspace_data, args=[PROJECT2_GUID])
         self._test_mv_file_and_triggering_loading_exception(
@@ -749,20 +749,20 @@ def _test_errors(self, url, fields, workspace_name, has_existing_data=False):
         self.mock_get_ws_access_level.assert_called_with(self.manager_user, TEST_WORKSPACE_NAMESPACE, workspace_name)
 
         # test missing columns
-        self.mock_load_file.return_value = [['family', 'individual'], ['1', '2']]
+        self._set_load_file_iter([['family', 'individual'], ['1', '2']])
         response = self.client.post(url, content_type='application/json', data=json.dumps(REQUEST_BODY))
         self.assertEqual(response.status_code, 400)
         response_json = response.json()
         self.assertListEqual(response_json['errors'], ['Missing required columns: Affected, HPO Terms, Sex'])
 
-        self.mock_load_file.return_value = LOAD_SAMPLE_DATA + MISSING_REQUIRED_SAMPLE_DATA
+        self._set_load_file_iter(LOAD_SAMPLE_DATA + MISSING_REQUIRED_SAMPLE_DATA)
         response = self.client.post(url, content_type='application/json', data=json.dumps(REQUEST_BODY))
         self.assertEqual(response.status_code, 400)
         response_json = response.json()
         self.assertListEqual(response_json['errors'], ['Missing Sex in row #4', 'Missing Affected in row #4'])
 
         # test sample data error and missing samples
-        self.mock_load_file.return_value = LOAD_SAMPLE_DATA + BAD_SAMPLE_DATA
+        self._set_load_file_iter(LOAD_SAMPLE_DATA + BAD_SAMPLE_DATA)
         response = self.client.post(url, content_type='application/json', data=json.dumps(REQUEST_BODY))
         self.assertEqual(response.status_code, 400)
         response_json = response.json()
@@ -780,7 +780,7 @@ def _test_errors(self, url, fields, workspace_name, has_existing_data=False):
             'NA19681 has invalid HPO terms: HP:0100258',
         ])
 
-        self.mock_load_file.return_value = LOAD_SAMPLE_DATA_NO_AFFECTED
+        self._set_load_file_iter(LOAD_SAMPLE_DATA_NO_AFFECTED)
         response = self.client.post(url, content_type='application/json', data=json.dumps(REQUEST_BODY))
         self.assertEqual(response.status_code, 400)
         response_json = response.json()
@@ -834,11 +834,13 @@ def _assert_valid_operation(self, project, test_add_data=True):
 
         gs_path = f'gs://seqr-loading-temp/v3.1/{genome_version}/SNV_INDEL/pedigrees/WES/'
         self.mock_subprocess.assert_has_calls([
+            mock.call('gsutil ls gs://seqr-scratch-temp/temp_upload_test_temp_file_id.json.gz', stdout=-1, stderr=-2, shell=True),  # nosec
+            mock.call().wait(),
+            mock.call('gsutil cat gs://seqr-scratch-temp/temp_upload_test_temp_file_id.json.gz | gunzip -c -q - ', stdout=-1, stderr=-2, shell=True),  # nosec
             mock.call(f'gsutil mv {TEMP_PATH}/* {gs_path}',  stdout=-1, stderr=-2, shell=True), # nosec
             mock.call().wait(),
             mock.call('gsutil ls gs://seqr-loading-temp/v3.1/db_id_to_gene_id.csv.gz', stdout=-1, stderr=-2, shell=True),  # nosec
             mock.call().wait(),
-            mock.call().stdout.__iter__(),
             mock.call(f'gsutil mv {TEMP_PATH}/* gs://seqr-loading-temp/v3.1/',  stdout=-1, stderr=-2, shell=True), # nosec
             mock.call().wait(),
         ])
@@ -855,7 +857,7 @@ def _assert_valid_operation(self, project, test_add_data=True):
         self._assert_expected_requests(variables, project, num_samples=14 if test_add_data else 3, status='Loading')
         self.assert_expected_airtable_headers(-1)
 
-        sample_summary = '13 new and 7 re-loaded' if test_add_data else '3 new'
+        sample_summary = '7 new and 7 re-loaded' if test_add_data else '3 new'
         self.mock_slack.assert_called_with(
             SEQR_SLACK_ANVIL_DATA_LOADING_CHANNEL,
             self._success_slack_message(project, sample_summary, genome_version, variables),
@@ -913,11 +915,11 @@ def _success_slack_message(self, project, sample_summary, genome_version, variab
 Loading pipeline is triggered with:
 ```{json.dumps(variables, indent=4)}```"""
 
-    @staticmethod
-    def _raise_move_file_error(command, *args, **kwargs):
+    def _raise_move_file_error(self, command, *args, **kwargs):
         mock_subprocess = mock.MagicMock()
-        mock_subprocess.wait.return_value = 1 if 'pedigrees' in command else 0
-        mock_subprocess.stdout = [b'Something wrong while moving the file.']
+        is_mv_pedigree_command = 'pedigrees' in command
+        mock_subprocess.wait.return_value = 1 if is_mv_pedigree_command else 0
+        mock_subprocess.stdout = [b'Something wrong while moving the file.'] if is_mv_pedigree_command else self.mock_subprocess.return_value.stdout
         return mock_subprocess
 
     def _test_mv_file_and_triggering_loading_exception(self, url, workspace, sample_data, genome_version, request_body, num_samples=None, sample_type='WES'):
@@ -959,7 +961,7 @@ def _test_mv_file_and_triggering_loading_exception(self, url, workspace, sample_
         self.mock_slack.assert_has_calls([
             mock.call(SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, slack_message_on_failure),
             mock.call(SEQR_SLACK_ANVIL_DATA_LOADING_CHANNEL, self._success_slack_message(
-                project, '3 new' if sample_type == 'WES' else '5 new and 1 re-loaded', genome_version, variables, sample_type,
+                project, '3 new' if sample_type == 'WES' else '4 new and 1 re-loaded', genome_version, variables, sample_type,
             )),
         ])
         self._assert_expected_requests(