Dev

seqr/management/commands/check_for_new_samples_from_pipeline.py

@@ -22,7 +22,7 @@
 from seqr.views.utils.permissions_utils import is_internal_anvil_project, project_has_anvil
 from seqr.views.utils.variant_utils import reset_cached_search_results, update_projects_saved_variant_json, \
     get_saved_variants
-from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, HAIL_SEARCH_DATA_DIR, ANVIL_UI_URL, \
+from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, PIPELINE_DATA_DIR, ANVIL_UI_URL, \
     SEQR_SLACK_ANVIL_DATA_LOADING_CHANNEL
 
 logger = logging.getLogger(__name__)
@@ -146,7 +146,7 @@ def _get_runs(cls, **kwargs):
     @staticmethod
     def _run_path(get_field_format):
         return RUN_FILE_PATH_TEMPLATE.format(
-            data_dir=HAIL_SEARCH_DATA_DIR,
+            data_dir=PIPELINE_DATA_DIR,
             **{field: get_field_format(field) for field in RUN_PATH_FIELDS}
         )
 

seqr/management/tests/check_for_new_samples_from_pipeline_tests.py

@@ -369,7 +369,7 @@ def set_up(self):
         mock_rand_int = patcher.start()
         mock_rand_int.side_effect = [GUID_ID, GUID_ID, GUID_ID, GUID_ID, GCNV_GUID_ID, GCNV_GUID_ID, GCNV_GUID_ID, GCNV_GUID_ID, GUID_ID, GUID_ID, GUID_ID, GUID_ID]
         self.addCleanup(patcher.stop)
-        patcher = mock.patch('seqr.management.commands.check_for_new_samples_from_pipeline.HAIL_SEARCH_DATA_DIR')
+        patcher = mock.patch('seqr.management.commands.check_for_new_samples_from_pipeline.PIPELINE_DATA_DIR')
         mock_data_dir = patcher.start()
         mock_data_dir.__str__.return_value = self.MOCK_DATA_DIR
         self.addCleanup(patcher.stop)

seqr/management/tests/update_individuals_sample_qc_tests.py

@@ -74,7 +74,7 @@ class UpdateIndividualsSampleQC(TestCase):
     fixtures = ['users', '1kg_project']
 
     def setUp(self):
-        patcher = mock.patch('seqr.management.commands.check_for_new_samples_from_pipeline.HAIL_SEARCH_DATA_DIR')
+        patcher = mock.patch('seqr.management.commands.check_for_new_samples_from_pipeline.PIPELINE_DATA_DIR')
         mock_data_dir = patcher.start()
         mock_data_dir.__str__.return_value = 'gs://seqr-hail-search-data/v3.1'
         self.addCleanup(patcher.stop)

seqr/utils/search/add_data_utils.py

@@ -3,9 +3,10 @@
 from django.db.models import F
 from typing import Callable
 
-from reference_data.models import GENOME_VERSION_LOOKUP
+from reference_data.models import GeneInfo, GENOME_VERSION_LOOKUP
 from seqr.models import Sample, Individual, Project
 from seqr.utils.communication_utils import send_project_notification
+from seqr.utils.file_utils import does_file_exist
 from seqr.utils.logging_utils import SeqrLogger
 from seqr.utils.search.utils import backend_specific_call
 from seqr.utils.search.elasticsearch.es_utils import validate_es_index_metadata_and_get_samples
@@ -117,7 +118,7 @@ def _format_loading_pipeline_variables(
     return variables
 
 def prepare_data_loading_request(projects: list[Project], individual_ids: list[int], sample_type: str, dataset_type: str, genome_version: str,
-                                 data_path: str, user: User, pedigree_dir: str,  raise_pedigree_error: bool = False,
+                                 data_path: str, user: User, load_data_dir: str,  raise_pedigree_error: bool = False,
                                  skip_validation: bool = False, skip_check_sex_and_relatedness: bool = False, vcf_sample_id_map=None):
     variables = _format_loading_pipeline_variables(
         projects,
@@ -130,8 +131,9 @@ def prepare_data_loading_request(projects: list[Project], individual_ids: list[i
         variables['skip_validation'] = True
     if skip_check_sex_and_relatedness:
         variables['skip_check_sex_and_relatedness'] = True
-    file_path = _get_pedigree_path(pedigree_dir, genome_version, sample_type, dataset_type)
+    file_path = _get_pedigree_path(load_data_dir, genome_version, sample_type, dataset_type)
     _upload_data_loading_files(individual_ids, vcf_sample_id_map or {}, user, file_path, raise_pedigree_error)
+    _write_gene_id_file(load_data_dir, user)
     return variables, file_path
 
 
@@ -172,6 +174,23 @@ def _upload_data_loading_files(individual_ids: list[int], vcf_sample_id_map: dic
             raise e
 
 
+def _write_gene_id_file(load_data_dir, user):
+    file_name = 'db_id_to_gene_id'
+    if does_file_exist(f'{load_data_dir}/{file_name}.csv.gz'):
+        return
+
+    gene_data_loaded = (GeneInfo.objects.filter(gencode_release=int(GeneInfo.CURRENT_VERSION)).exists() and
+                        GeneInfo.objects.filter(gencode_release=int(GeneInfo.ALL_GENCODE_VERSIONS[-1])).exists())
+    if not gene_data_loaded:
+        raise ValueError(
+            'Gene reference data is not yet loaded. If this is a new seqr installation, wait for the initial data load '
+            'to complete. If this is an existing installation, see the documentation for updating data in seqr.'
+        )
+    gene_data = GeneInfo.objects.all().values('gene_id', db_id=F('id')).order_by('id')
+    file_config = (file_name, ['db_id', 'gene_id'], gene_data)
+    write_multiple_files([file_config], load_data_dir, user, file_format='csv', gzip_file=True)
+
+
 def _get_pedigree_path(pedigree_dir: str, genome_version: str, sample_type: str, dataset_type: str):
     dag_dataset_type = _dag_dataset_type(sample_type, dataset_type)
     return f'{pedigree_dir}/{GENOME_VERSION_LOOKUP[genome_version]}/{dag_dataset_type}/pedigrees/{sample_type}'

seqr/views/apis/anvil_workspace_api_tests.py

@@ -505,6 +505,7 @@ def _test_get_workspace_files(self, url, response_key, expected_files, mock_subp
         ])
 
 
+@mock.patch('reference_data.models.GeneInfo.CURRENT_VERSION', '27')
 class LoadAnvilDataAPITest(AirflowTestCase, AirtableTest):
     fixtures = ['users', 'social_auth', 'reference_data', '1kg_project']
 
@@ -561,6 +562,9 @@ def setUp(self):
         patcher = mock.patch('seqr.views.utils.export_utils.open')
         self.mock_temp_open = patcher.start()
         self.addCleanup(patcher.stop)
+        patcher = mock.patch('seqr.views.utils.export_utils.gzip.open')
+        self.mock_gzip_temp_open = patcher.start()
+        self.addCleanup(patcher.stop)
         patcher = mock.patch('seqr.views.apis.anvil_workspace_api.logger')
         self.mock_api_logger = patcher.start()
         self.addCleanup(patcher.stop)
@@ -796,10 +800,16 @@ def _assert_valid_operation(self, project, test_add_data=True):
             '\n'.join(['\t'.join(row) for row in [header] + rows])
         )
 
+        self.mock_gzip_temp_open.assert_called_with(f'{TEMP_PATH}/db_id_to_gene_id.csv.gz', 'w')
+        gene_file = self.mock_gzip_temp_open.return_value.__enter__.return_value.write.call_args.args[0].split('\n')
+        self.assertEqual(len(gene_file), 52)
+        self.assertListEqual(gene_file[:3], ['db_id,gene_id', '1,ENSG00000223972', '2,ENSG00000227232'])
+
         gs_path = f'gs://seqr-loading-temp/v3.1/{genome_version}/SNV_INDEL/pedigrees/WES/'
-        self.mock_mv_file.assert_called_with(
-            f'{TEMP_PATH}/*', gs_path, self.manager_user
-        )
+        self.mock_mv_file.assert_has_calls([
+            mock.call(f'{TEMP_PATH}/*', gs_path, self.manager_user),
+            mock.call(f'{TEMP_PATH}/*', 'gs://seqr-loading-temp/v3.1/', self.manager_user)
+        ])
 
         self.assert_airflow_loading_calls(additional_tasks_check=test_add_data)
 
@@ -866,11 +876,16 @@ def _assert_valid_operation(self, project, test_add_data=True):
             'father__individual_id': None, 'sex': 'F', 'affected': 'N', 'notes': 'a individual note', 'features': [],
         }, individual_model_data)
 
+    @staticmethod
+    def _raise_move_file_error(from_path, to_path, *args, **kwargs):
+        if 'pedigrees' in to_path:
+            raise Exception('Something wrong while moving the file.')
+
     def _test_mv_file_and_triggering_dag_exception(self, url, workspace, sample_data, genome_version, request_body, num_samples=None, sample_type='WES'):
         # Test saving ID file exception
         responses.calls.reset()
         self.mock_authorized_session.reset_mock()
-        self.mock_mv_file.side_effect = Exception('Something wrong while moving the file.')
+        self.mock_mv_file.side_effect = self._raise_move_file_error
         # Test triggering dag exception
         self.set_dag_trigger_error_response()
 

seqr/views/apis/data_manager_api.py

@@ -354,7 +354,7 @@ def load_data(request):
         )
     else:
         request_json, _ = prepare_data_loading_request(
-            *loading_args, **loading_kwargs, pedigree_dir=LOADING_DATASETS_DIR, raise_pedigree_error=True,
+            *loading_args, **loading_kwargs, load_data_dir=LOADING_DATASETS_DIR, raise_pedigree_error=True,
         )
         response = requests.post(f'{PIPELINE_RUNNER_SERVER}/loading_pipeline_enqueue', json=request_json, timeout=60)
         if response.status_code == 409: