Dev

.github/workflows/unit-tests.yml

@@ -99,7 +99,7 @@ jobs:
           export CLICKHOUSE_READER_USER=clickhouse_read_only
           export CLICKHOUSE_WRITER_USER=clickhouse_test
           export CLICKHOUSE_SERVICE_HOSTNAME=localhost
-          coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data seqr matchmaker panelapp clickhouse_search
+          coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data clickhouse_search seqr matchmaker panelapp
           coverage report -m --fail-under=99
           coverage report | (! grep [1-8][0-9]%)
           coverage report | (! grep 9[0-5]%)

clickhouse_search/search_tests.py → clickhouse_search/all_search_tests.py

@@ -38,7 +38,7 @@
     DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, \
     format_cached_variant
 from reference_data.models import Omim
-from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults, SavedVariant, Individual
+from seqr.models import Project, Family, Dataset, VariantSearch, VariantSearchResults, SavedVariant, Individual
 from seqr.views.apis.data_manager_api import trigger_delete_project
 from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, \
     SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \
@@ -139,7 +139,7 @@ def assert_cached_results(self, expected_results, cache_key):
         self.mock_redis.expire.reset_mock()
 
     def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=None, request_body=None, check_login=None, query_params=None, search_hash=None, **search_kwargs):
-        search_hash = search_hash or random.randint(1000, 10000)  # nosec
+        search_hash = search_hash or random.randint(1000, 9000000)  # nosec
         self.mock_results_guid.return_value = f'VRS{search_hash:07d}'
         url = reverse(query_variants_handler, args=[search_hash])
 
@@ -250,8 +250,6 @@ def _get_cached_variant(cls, variant, cached_variant_fields):
 
     def _set_grch37_search(self):
         Project.objects.filter(id=1).update(genome_version='37')
-        Sample.objects.filter(sample_id='HG00732').update(is_active=False)
-        Sample.objects.exclude(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS).update(is_active=False)
 
     def test_single_family_search(self):
         variant_gene_counts = {
@@ -326,7 +324,7 @@ def test_single_project_search(self):
             gene_counts={**variant_gene_counts, **GCNV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}}
         )
 
-        self._add_sample_type_samples('WES', dataset_type='SV', guid__in=['S000135_na20870'])
+        self._add_sample_type_samples('WES', dataset_type='SV', guid__in=['S000129_na19675'])
         self._assert_expected_search(
             [GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], gene_counts={
                 'ENSG00000129562': {'total': 1, 'families': {'F000002_2': 1, 'F000003_3': 1}},
@@ -410,17 +408,21 @@ def test_all_project_search(self):
         )
 
     def test_both_sample_types_search(self):
-        Sample.objects.filter(dataset_type='MITO').update(is_active=False)
+        Dataset.objects.get(guid='S000149_hg00733').active_individuals.clear()
 
         # One family (F000011_11) in a multi-project search has identical exome and genome data.
-        self._add_sample_type_samples('WES', individual__family__guid='F000011_11')
+        self._add_sample_type_samples('WES', active_individuals__family__guid='F000011_11')
 
         self._assert_expected_search(
             MULTI_PROJECT_BOTH_SAMPLE_TYPE_VARIANTS, gene_counts=GENE_COUNTS, locus={'rawItems': 'chr1:1-100000000'},
             project_families=MULTI_PROJECT_PROJECT_FAMILIES, check_login=self.check_collaborator_login,
         )
 
-        self._add_sample_type_samples('WGS', guid__in=['S000132_hg00731'])
+        dataset = Dataset.objects.get(guid='S000129_na19675')
+        dataset.pk = None
+        dataset.sample_type = 'WGS'
+        dataset.save()
+        dataset.active_individuals.add(4)
 
         # Variant 1 is de novo in exome but inherited and homozygous in genome.
         # Variant 2 is inherited and homozygous in exome and de novo and homozygous in genome, so it fails de-novo inheritance when parental data is missing in genome.
@@ -437,12 +439,13 @@ def test_both_sample_types_search(self):
             inheritance_mode='any_affected', quality_filter={'min_gq': 40, 'min_qs': 20}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES,
         )
 
+        self.maxDiff = None
         self._assert_expected_search(
             [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1],
             inheritance_mode='de_novo', quality_filter=None, project_families=SINGLE_FAMILY_PROJECT_FAMILIES,
         )
 
-        self._add_sample_type_samples('WGS', guid__in=['S000133_hg00732', 'S000134_hg00733'])
+        dataset.active_individuals.add(5, 6)
         self._assert_expected_search(
             [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT3_BOTH_SAMPLE_TYPES,
              VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4],
@@ -459,7 +462,6 @@ def test_both_sample_types_search(self):
             inheritance_mode='de_novo', quality_filter={'min_gq': 40}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES,
         )
 
-        self.maxDiff = None
         self._assert_expected_search(
             [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES,
              [{**VARIANT2_BOTH_SAMPLE_TYPES, 'selectedMainTranscriptId': 'ENST00000450625'}, GCNV_VARIANT4],
@@ -474,12 +476,14 @@ def test_both_sample_types_search(self):
 
     @staticmethod
     def _add_sample_type_samples(sample_type, dataset_type=None, **sample_filter):
-        for sample in Sample.objects.filter(**sample_filter):
-            sample.pk = None
-            sample.sample_type = sample_type
+        for dataset in Dataset.objects.filter(**sample_filter):
+            individuals = dataset.active_individuals.all()
+            dataset.pk = None
+            dataset.sample_type = sample_type
             if dataset_type:
-                sample.dataset_type = dataset_type
-            sample.save()
+                dataset.dataset_type = dataset_type
+            dataset.save()
+            dataset.active_individuals.set(individuals)
 
     def test_inheritance_filter(self):
         inheritance_mode = 'any_affected'
@@ -647,10 +651,12 @@ def test_inheritance_filter(self):
         )
 
         # Test deletion in trans with hom alt snp/indel
-        for sample in Sample.objects.filter(individual__family_id=14):
-            sample.pk = None
-            sample.dataset_type = 'SNV_INDEL'
-            sample.save()
+        dataset = Dataset.objects.get(guid='S000147_na21234')
+        dataset_individuals = dataset.active_individuals.all()
+        dataset.pk = None
+        dataset.dataset_type = 'SNV_INDEL'
+        dataset.save()
+        dataset.active_individuals.set(dataset_individuals)
         self._assert_expected_search(
             [[SV_VARIANT1, SV_VARIANT2], [SV_VARIANT1, PROJECT_4_COMP_HET_VARIANT], PROJECT_4_COMP_HET_VARIANT, SV_VARIANT4],
             inheritance_mode=inheritance_mode, inheritance_filter=sv_affected, **COMP_HET_ALL_PASS_FILTERS,
@@ -678,7 +684,10 @@ def test_exclude_previous_search_results(self):
 
         request_body = {'projectFamilies': DEFAULT_PROJECT_FAMILIES, 'previousSearchHash': 'abc1234'}
         exclude = {'previousSearch': True}
-        response_search = {'exclude': {**exclude, 'previousSearchHash': 'abc1234'}}
+        response_search = {
+            'exclude_keys': {'SNV_INDEL': [1, 2]},
+            'exclude_key_pairs': {'SNV_INDEL': [[2, 3]],'SV_WES': [[18, 19]]},
+        }
         self._assert_expected_search(
             [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3],
             inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS,
@@ -692,6 +701,10 @@ def test_exclude_previous_search_results(self):
         self.set_cache(cache_key, [
             [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3,
         ])
+        response_search = {
+            'exclude_keys': {'MITO': [8], 'SV_WES': [18]},
+            'exclude_key_pairs': {'SNV_INDEL': [[3, 4]], 'SNV_INDEL,SV_WES': [[2, 19]]},
+        }
         self._assert_expected_search(
             [VARIANT2, [GCNV_VARIANT3, GCNV_VARIANT4]], exclude=exclude, **COMP_HET_ALL_PASS_FILTERS,
             request_body=request_body, response_search=response_search, inheritance_mode='recessive', cached_variant_fields=[
@@ -838,7 +851,7 @@ def test_location_search(self):
             ],
         )
 
-        self._add_sample_type_samples('WES', individual__family__guid='F000014_14')
+        self._add_sample_type_samples('WES', active_individuals__family__guid='F000014_14')
         self._assert_expected_search(
             [SV_VARIANT1, SV_VARIANT2, MULTI_PROJECT_GCNV_VARIANT3, GCNV_VARIANT4], locus=sv_locus,
             project_families=[*SINGLE_FAMILY_PROJECT_FAMILIES, *SV_PROJECT_FAMILIES],
@@ -858,7 +871,7 @@ def test_variant_id_search(self):
             [],locus={'rawVariantItems': VARIANT_IDS[1]},
         )
 
-    @mock.patch('seqr.utils.search.utils.MAX_EXPORT_VARIANTS', 2)
+    @mock.patch('seqr.views.apis.variant_search_api.MAX_EXPORT_VARIANTS', 2)
     @mock.patch('clickhouse_search.search.MAX_GENES_FOR_FILTER', 2)
     @mock.patch('clickhouse_search.search.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1)
     @mock.patch('clickhouse_search.search.MAX_VARIANTS', 3)
@@ -961,7 +974,7 @@ def test_invalid_search(self):
 
         self._assert_expected_search_error('Location must be specified to search across multiple projects', project_families=MULTI_PROJECT_PROJECT_FAMILIES)
 
-        Sample.objects.filter(guid='S000143_na20885').update(sample_id='HG00732')
+        Individual.objects.filter(guid='I000015_na20885').update(individual_id='HG00732')
         self._assert_expected_search_error(
             'The following samples are incorrectly configured and have different affected statuses in different projects: '
             'HG00732 (1kg project nåme with uniçøde/ Test Reprocessed Project)',
@@ -981,7 +994,7 @@ def test_invalid_search(self):
         response = self.client.get(export_url)
         self.assertEqual(response.status_code, 403)
 
-    @mock.patch('seqr.utils.search.utils.LiftOver')
+    @mock.patch('clickhouse_search.search.LiftOver')
     def test_variant_lookup(self, mock_liftover):
         mock_convert_coordinate = mock_liftover.return_value.convert_coordinate
         mock_convert_coordinate.side_effect = lambda chrom, pos: [(chrom, pos + 10000)]
@@ -1845,7 +1858,7 @@ def test_secondary_annotations_filter(self):
             ],
         )
 
-        self._add_sample_type_samples('WES', individual__family__guid='F000014_14')
+        self._add_sample_type_samples('WES', active_individuals__family__guid='F000014_14')
         self._assert_expected_search(
             [MULTI_DATA_TYPE_COMP_HET_VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], MULTI_PROJECT_GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]],
             inheritance_mode='recessive',
@@ -1869,7 +1882,6 @@ def test_secondary_annotations_filter(self):
             'familyGuids': ['F000002_2_x'],
             'genotypes': {k: {**v, 'familyGuid': 'F000002_2_x'} for k, v in MULTI_DATA_TYPE_COMP_HET_VARIANT2['genotypes'].items()}
         }
-        Sample.objects.filter(guid='S000146_hg00732').update(is_active=False)
         Family.objects.filter(guid='F000002_2').update(guid='F000002_2_x')
         self._assert_expected_search(
             [[missing_gt_comp_het_variant, missing_gt_gcnv_variant]],
@@ -2284,22 +2296,22 @@ def test_search_context(self):
         self.assertDictEqual(
             response_json['rnaSeqData']['I000001_na19675']['outliers']['ENSG00000268903'][0],
             {'geneId': 'ENSG00000268903', 'isSignificant': True, 'pAdjust': 1.39e-09, 'pValue': 5.88e-10,
-             'tissueType': 'M', 'zScore': 7.08}
+             'tissueType': 'M', 'sequencingType': 'T', 'zScore': 7.08}
         )
         self.assertListEqual(
             sorted(response_json['rnaSeqData']['I000001_na19675']['spliceOutliers']['ENSG00000268903'], key=lambda d: d['start']),
             [{'chrom': '7', 'counts': 1297, 'end': 4000, 'geneId': 'ENSG00000268903', 'isSignificant': True,
               'meanCounts': 0.85,  'meanTotalCounts': 0.85, 'pAdjust': 0.0003,
               'pValue': 0.0001, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297,
-              'start': 3000, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': -12.34},
+              'start': 3000, 'strand': '*', 'tissueType': 'F', 'sequencingType': 'W', 'type': 'psi5', 'deltaIntronJaccardIndex': -12.34},
              {'chrom': '7', 'counts': 1297, 'end': 8000, 'geneId': 'ENSG00000268903', 'isSignificant': True,
               'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 0.003,
               'pValue': 0.001, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297,
-              'start': 7000, 'strand': '*', 'tissueType': 'M', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34},
+              'start': 7000, 'strand': '*', 'tissueType': 'M', 'sequencingType': 'T', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34},
              {'chrom': '7', 'counts': 1297, 'end': 132886973, 'geneId': 'ENSG00000268903', 'isSignificant': True,
               'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 3.08e-56,
               'pValue': 1.08e-56, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297,
-              'start': 132885746, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}]
+              'start': 132885746, 'strand': '*', 'tissueType': 'F', 'sequencingType': 'W', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}]
         )
         self.assertDictEqual(response_json['savedVariantsByGuid'], {'SV0000001_2103343353_r0390_100': {
             **{k: mock.ANY for k in [
@@ -2392,6 +2404,7 @@ def test_search_context(self):
             'category': 'CMG Discovery Tags',
             'color': '#03441E',
             'searchHash': None,
+            'searchName': None,
             'metadata': None,
             'lastModifiedDate': '2018-05-29T16:32:51.449Z',
             'createdBy': None,
@@ -2460,9 +2473,10 @@ def test_trigger_delete_project(self):
             23: {'ac_wes': 0, 'ac_wgs': 0, 'hom_wes': 0, 'hom_wgs': 0, 'ac_affected': 0, 'hom_affected': 0},
         })
 
-        project_samples = Sample.objects.filter(individual__family__project__guid='R0001_1kg', is_active=True)
-        self.assertEqual(project_samples.filter(dataset_type='SNV_INDEL').count(), 0)
-        self.assertEqual(project_samples.count(), 4)
+        project_datasets = Dataset.objects.filter(active_individuals__family__project__guid='R0001_1kg')
+        self.assertEqual(project_datasets.filter(dataset_type='SNV_INDEL').count(), 0)
+        self.assertEqual(project_datasets.count(), 4)
+        self.assertEqual(Dataset.objects.get(guid='S000129_na19675').inactive_individuals.count(), 7)
 
         body['datasetType'] = 'SV'
         response = self.client.post(url, content_type='application/json', data=json.dumps(body))
@@ -2473,7 +2487,7 @@ def test_trigger_delete_project(self):
                 'Deleted all GCNV search data for project 1kg project n\xe5me with uni\xe7\xf8de',
             ],
         })
-        self.assertEqual(project_samples.filter(dataset_type='SV').count(), 0)
-        self.assertEqual(project_samples.count(), 1)
+        self.assertEqual(project_datasets.filter(dataset_type='SV').count(), 0)
+        self.assertEqual(project_datasets.count(), 1)
 
 

clickhouse_search/backend/table_models.py

@@ -3,6 +3,10 @@
 from django.db.models import Func
 
 from clickhouse_search.backend.fields import NamedTupleField
+from seqr.utils.logging_utils import SeqrLogger
+
+logger = SeqrLogger(__name__)
+
 
 MATERIALIZED_VIEW_META_FIELDS = [
     'to_table', 'source_table', 'source_sql', 'source_url', 'source_url_template', 'column_selects', 'refreshable', 'create_empty',
@@ -40,7 +44,8 @@ class Meta:
 class RefreshableMaterializedView(IncrementalMaterializedView):
 
     @classmethod
-    def refresh(cls):
+    def refresh(cls, user=None):
+        logger.info(f'Refreshing materialized view {cls._meta.db_table}', user)
         with connections['clickhouse_write'].cursor() as cursor:
             cursor.execute(f'SYSTEM START VIEW "{cls._meta.db_table}"')
             cursor.execute(f'SYSTEM REFRESH VIEW "{cls._meta.db_table}"')
@@ -88,7 +93,8 @@ def dict_get_expression(cls, *expressions, field_names=None, force_tuple=False,
         return dict_get_func
 
     @classmethod
-    def reload(cls):
+    def reload(cls, user=None):
+        logger.info(f'Reloading dictionary {cls._meta.db_table}', user)
         with connections['clickhouse_write'].cursor() as cursor:
             cursor.execute(f'SYSTEM RELOAD DICTIONARY "{cls._meta.db_table}"')