Dev

clickhouse_search/managers.py

@@ -1546,7 +1546,7 @@ def _prefilter_entries(self, entries, freqs=None, **kwargs):
 
     def _filter_frequency(self, entries, frequencies, pathogenicity=None, sample_data=None, **kwargs):
         callset_filter = frequencies.get(self.callset_filter_field) or {}
-        if (not callset_filter.get('ac') or callset_filter['ac'] >= self.MIN_MULTI_FAMILY_SEQR_AC) and (
+        if (not callset_filter.get('ac') or callset_filter['ac'] > self.MIN_MULTI_FAMILY_SEQR_AC) and (
             len(sample_data['sample_type_families']) > 1 or len(next(iter(sample_data['sample_type_families'].values()))) > 1
         ):
             raise InvalidSearchException(

clickhouse_search/migrations/0040_gnomadnoncodingconstraintdict.py

@@ -418,7 +418,7 @@ class Migration(migrations.Migration):
             ],
             options={
                 'db_table': 'GRCh38/SNV_INDEL/entries',
-                'engine': clickhouse_search.backend.engines.CollapsingMergeTree('sign', deduplicate_merge_projection_mode='rebuild', index_granularity=8192, order_by=('project_guid', 'family_guid', 'sample_type', 'is_gnomad_gt_5_percent', 'is_annotated_in_any_gene', 'key'), partition_by='project_guid'),
+                'engine': clickhouse_search.backend.engines.CollapsingMergeTree('sign', deduplicate_merge_projection_mode='rebuild', index_granularity=8192, order_by=('project_guid', 'family_guid', 'sample_type', 'is_gnomad_gt_5_percent', 'is_annotated_in_any_gene', 'key'), partition_by=['project_guid', 'partition_id']),
                 'projection': clickhouse_search.models.search_models.Projection('xpos_projection', order_by='is_gnomad_gt_5_percent, is_annotated_in_any_gene, xpos'),
             },
             managers=[

clickhouse_search/models/search_models.py

@@ -359,7 +359,7 @@ class Meta:
         engine = CollapsingMergeTree(
             'sign',
             order_by=('project_guid', 'family_guid', 'sample_type', 'is_gnomad_gt_5_percent', 'is_annotated_in_any_gene', 'key'),
-            partition_by='project_guid, partition_id',
+            partition_by=['project_guid', 'partition_id'],
             deduplicate_merge_projection_mode='rebuild',
             index_granularity=8192,
         )

clickhouse_search/search.py

@@ -5,6 +5,7 @@
 from django.db import connections
 from django.db.models import Count, F, Min, Q
 from django.db.models.functions import JSONObject
+from django.db.utils import OperationalError
 import json
 
 from clickhouse_search.backend.fields import NamedTupleField
@@ -13,9 +14,9 @@
 from clickhouse_search.managers import InvalidDatasetTypeException, InvalidSearchException
 from clickhouse_search.models.gt_stats_models import PROJECT_GT_STATS_VIEW_CLASS_MAP
 from clickhouse_search.models.reference_data_models import BaseClinvar, BaseHgmd
-from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, \
+from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, EntriesSnvIndel,  \
     ENTRY_CLASS_MAP, VARIANTS_CLASS_MAP, VARIANT_DETAILS_CLASS_MAP
-from reference_data.models import GeneInfo, GeneConstraint, Omim, GENOME_VERSION_LOOKUP
+from reference_data.models import GeneInfo, GeneConstraint, Omim, GENOME_VERSION_LOOKUP, GENOME_VERSION_GRCh38
 from seqr.models import Sample, PhenotypePrioritization, Family, Individual
 from seqr.utils.gene_utils import parse_locus_list_items
 from seqr.utils.logging_utils import SeqrLogger
@@ -1030,8 +1031,20 @@ def delete_clickhouse_project(project, dataset_type, sample_type=None):
     if dataset_type == Sample.DATASET_TYPE_SV_CALLS and sample_type == Sample.SAMPLE_TYPE_WES:
         dataset_type = 'GCNV'
     table_base = f'{GENOME_VERSION_LOOKUP[project.genome_version]}/{dataset_type}'
+    partition_id = f"'{project.guid}'"
+    partition_ids = [partition_id]
+    if project.genome_version == GENOME_VERSION_GRCh38 and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS:
+        partition_ids = [
+            f'({partition_id}, {pid})' for pid in
+            EntriesSnvIndel.objects.filter(project_guid=project.guid).values_list('partition_id', flat=True).distinct()
+        ]
     with connections['clickhouse_write'].cursor() as cursor:
-        cursor.execute(f'ALTER TABLE "{table_base}/entries" DROP PARTITION %s', [project.guid])
+        for partition_id in partition_ids:
+            try:
+                cursor.execute(f"ALTER TABLE `{table_base}/entries` DROP PARTITION {partition_id}")
+            except OperationalError:
+                # Repartitioning of this table was not rolled out via a standard migration and may be skipped in some local installs
+                raise OperationalError('Unable to delete project data due to a partitioning migration error. Please reach out to the seqr team for assistance.')
         if dataset_type != 'GCNV':
             cursor.execute(f'ALTER TABLE "{table_base}/project_gt_stats" DROP PARTITION %s', [project.guid])
             PROJECT_GT_STATS_VIEW_CLASS_MAP[project.genome_version][dataset_type].refresh()

clickhouse_search/search_tests.py

@@ -168,14 +168,22 @@ def _assert_expected_search(self, expected_results, results_page=None, gene_coun
         response, search_hash, search_body = self._execute_search(project_families=project_families, sort=sort, **kwargs)
         self.assertEqual(response.status_code, 200)
         expected_response = {
-            'searchedVariants': results_page or expected_results,
+            'searchedVariantIds': [],
+            'variantsById': {},
             'search': {
                 'search': {**search_body, **(response_search or {})},
                 'projectFamilies': DEFAULT_PROJECT_FAMILIES if project_families is None else project_families,
                 'totalResults': len(expected_results),
             },
 
         }
+        for variant in results_page or expected_results:
+            if isinstance(variant, list):
+                expected_response['searchedVariantIds'].append([v['variantId'] for v in variant])
+                expected_response['variantsById'].update({v['variantId']: v for v in variant})
+            else:
+                expected_response['searchedVariantIds'].append(variant['variantId'])
+                expected_response['variantsById'][variant['variantId']] = variant
         if expected_results:
             expected_response.update({
                 **{key: mock.ANY for key in SEARCH_RESPONSE_KEYS},
@@ -214,7 +222,7 @@ def _assert_expected_export_data(self, search_hash, export_data):
         url = reverse(export_variants_handler, args=[search_hash])
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
-        self.assertListEqual([line.split('\t') for line in response.content.decode().strip().split('\n')], export_data)
+        self.assertListEqual([line.split('\t') for line in response.content.decode().strip('\n').split('\n')], export_data)
 
     def _assert_expected_search_error(self, error, **kwargs):
         response, search_hash, _ = self._execute_search(**kwargs)
@@ -359,8 +367,12 @@ def test_multi_project_search(self):
         )
 
     def test_all_project_search(self):
-        self.login_collaborator()
         request_body = {'allGenomeProjectFamilies': '38'}
+        self._assert_expected_search_error(
+            'No data available for genome version "GRCh38"', request_body=request_body, check_login=self.check_require_login,
+        )
+
+        self.login_collaborator()
         project_families = [{'projectGuid': 'R0001_1kg', 'familyGuids': [
             'F000001_1', 'F000002_2', 'F000003_3', 'F000004_4', 'F000005_5', 'F000006_6', 'F000007_7', 'F000008_8',
             'F000009_9', 'F000010_10', 'F000013_13',
@@ -1203,11 +1215,11 @@ def test_variant_lookup(self, mock_liftover):
         }])
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
-        self.assertDictEqual(response.json()['variants'][0], {
+        self.assertDictEqual(response.json()['variantsById'], {'1-10439-AC-A': {
             **VARIANT1,
             'familyGuids': [],
             'lookupFamilyGuids': VARIANT1['familyGuids'],
-        })
+        }})
 
         cache_key = 'variant_lookup_results__7-143270172-A-G__37'
         self._assert_expected_lookup(
@@ -1326,11 +1338,11 @@ def _assert_expected_lookup(self, variant_id, variant, cache_key, genome_version
             'individualsByGuid': expected_individuals or {i: mock.ANY for i in individual_guids or []},
             'locusListsByGuid': {'LL00049_pid_genes_autosomal_do': mock.ANY, 'LL00005_retina_proteome': mock.ANY},
             'totalSampleCounts': {'MITO': {'WES': 1}, 'SNV_INDEL': {'WES': 7}, 'SV': {'WES': 3, 'WGS': 3}} if genome_version == '38' else {},
-            'variants': [{
+            'variantsById': {v['variantId']: {
                 **v,
                 'familyGuids': [],
                 'lookupFamilyGuids': v['familyGuids'] + v.get('liftedFamilyGuids', []),
-            } for v in variants],
+            } for v in variants},
             **kwargs,
         }
         self.assertDictEqual(response.json(), expected_body)
@@ -1362,7 +1374,8 @@ def test_get_single_variant(self):
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
         self.assertDictEqual(response.json(), {
-            'searchedVariants': [FAMILY_1_VARIANT],
+            'searchedVariantIds': ['21-3343353-GAGA-G'],
+            'variantsById': {'21-3343353-GAGA-G': FAMILY_1_VARIANT},
             'familiesByGuid': {'F000001_1': mock.ANY},
             'projectsByGuid': {'R0001_1kg': mock.ANY},
             'locusListsByGuid': {'LL00049_pid_genes_autosomal_do': {
@@ -1379,21 +1392,21 @@ def test_get_single_variant(self):
 
         response = self.client.get(url_template.format('M-4429-G-A', 'F000002_2'))
         self.assertEqual(response.status_code, 200)
-        self.assertListEqual(response.json()['searchedVariants'], [MITO_VARIANT1])
+        self.assertDictEqual(response.json()['variantsById'], {'M-4429-G-A': MITO_VARIANT1})
 
         response = self.client.get(url_template.format('suffix_140608_DUP', 'F000002_2'))
         self.assertEqual(response.status_code, 200)
-        self.assertListEqual(response.json()['searchedVariants'], [GCNV_VARIANT4])
+        self.assertDictEqual(response.json()['variantsById'], {'suffix_140608_DUP': GCNV_VARIANT4})
 
         self.login_manager()
         response = self.client.get(url_template.format('phase2_DEL_chr14_4640', 'F000014_14'))
         self.assertEqual(response.status_code, 200)
-        self.assertListEqual(response.json()['searchedVariants'], [SV_VARIANT4])
+        self.assertDictEqual(response.json()['variantsById'], {'phase2_DEL_chr14_4640': SV_VARIANT4})
 
         self._set_grch37_search()
         response = self.client.get(url_template.format('7-143270172-A-G', 'F000002_2'))
         self.assertEqual(response.status_code, 200)
-        self.assertListEqual(response.json()['searchedVariants'], [GRCH37_VARIANT])
+        self.assertDictEqual(response.json()['variantsById'], {'7-143270172-A-G': GRCH37_VARIANT})
 
         self.mock_redis.get.assert_not_called()
         self.mock_redis.set.assert_not_called()
@@ -2137,7 +2150,9 @@ def test_gene_variant_lookup(self):
         self._assert_expected_search(
             [variant4], request_body=request_body, response_search=response_search,
             cached_variant_fields=[{'selectedTranscript': CACHED_CONSEQUENCES_BY_KEY[4][1]}],
-            annotations=annotations, freqs=freqs, locus=locus, project_families=[],
+            annotations=annotations, freqs=freqs, locus=locus, project_families=[], export_data=[
+                EXPORT_DATA[0][:27], EXPORT_DATA[4][:24] + ['3 Families', '', ''],
+            ],
         )
 
         freqs = {'callset': freqs['callset']}
@@ -2357,16 +2372,16 @@ def test_search_context(self):
         }])
         response, _, _ = self._execute_search(search_hash=9876)
         self.assertEqual(response.status_code, 200)
-        variants = response.json()['searchedVariants']
+        variants = response.json()['variantsById']
         self.assertEqual(len(variants), 1)
-        self.assertFalse('discoveryTags' in variants[0])
+        self.assertFalse('discoveryTags' in variants['1-248367227-TC-T'])
 
         self.login_analyst_user()
         response, _, _ = self._execute_search(search_hash=9876)
         self.assertEqual(response.status_code, 200)
-        variants = response.json()['searchedVariants']
+        variants = response.json()['variantsById']
         self.assertEqual(len(variants), 1)
-        self.assertListEqual(variants[0]['discoveryTags'], [{
+        self.assertListEqual(variants['1-248367227-TC-T']['discoveryTags'], [{
             'savedVariant': {
                 'variantGuid': 'SV0000006_1248367227_r0003_tes',
                 'familyGuid': 'F000012_12',

clickhouse_search/test_utils.py

@@ -1464,7 +1464,7 @@ def format_cached_variant(variant):
      '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''],
     ['M', '14783', 'T', 'C', 'ENSG00000198727', '', 'synonymous_variant', '1', '', '', '', '', '', '', '', '', '', '', '',
      'rs193302982', 'ENST00000361789.2:c.37T>C', 'ENSP00000354554.2:p.Leu13=', 'Likely_pathogenic', '0', '2', '', '', '',
-     '', '', 'HG00731', '2', 'artifact_prone_site'],
+     '', '', 'HG00731', '2', 'artifact_prone_site', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', ''],
 ]
 
 SPLIT_FAMILY_EXPORT_DATA = [

matchmaker/views/matchmaker_api.py

@@ -2,7 +2,7 @@
 import requests
 from datetime import datetime
 from django.core.mail.message import EmailMessage
-from django.db.models import prefetch_related_objects, Q, Value, CharField
+from django.db.models import prefetch_related_objects, F, Q, Value, CharField
 from django.db.models.functions import Coalesce
 
 from matchmaker.models import MatchmakerResult, MatchmakerContactNotes, MatchmakerSubmission, MatchmakerSubmissionGenes, \
@@ -58,14 +58,16 @@ def get_individual_mme_matches(request, submission_guid):
     variant_guids = {gv['variantGuid'] for gv in gene_variants}
 
     variants = get_json_for_saved_variants(
-        SavedVariant.objects.filter(guid__in=variant_guids), genome_version=project.genome_version, additional_values={
+        SavedVariant.objects.filter(guid__in=variant_guids), additional_values={
             'genomeVersion': Coalesce('saved_variant_json__genomeVersion', Value(project.genome_version), output_field=CharField()),
+            'selectedMainTranscript': F('main_transcript'),
+            'xposEnd': F('xpos_end'),
         },
     )
     response_json['savedVariantsByGuid'] = {}
     for variant in variants:
-        chrom, pos = get_chrom_pos(variant['xpos'])
-        variant.update({'chrom': chrom, 'pos': pos})
+        _, end = get_chrom_pos(variant.pop('xposEnd'))
+        variant.update({'end': end})
         response_json['savedVariantsByGuid'][variant['variantGuid']] = variant
 
     return _parse_mme_results(

matchmaker/views/matchmaker_api_tests.py

@@ -231,7 +231,7 @@ def test_get_individual_mme_matches(self):
             set(response_json['savedVariantsByGuid'].keys()),
             {'SV0000001_2103343353_r0390_100'}
         )
-        saved_variant_fields = {*SAVED_VARIANT_FIELDS, 'chrom', 'pos', 'genomeVersion'}
+        saved_variant_fields = {*SAVED_VARIANT_FIELDS, 'selectedMainTranscript', 'end', 'genomeVersion'}
         self.assertSetEqual(set(response_json['savedVariantsByGuid']['SV0000001_2103343353_r0390_100'].keys()), saved_variant_fields)
         self.assertDictEqual(response_json['mmeContactNotes'], {})
 

seqr/management/commands/tag_seqr_prioritized_variants.py

@@ -240,6 +240,7 @@
             'family_filter': {
                 CONFIRMED_FAMILY_FILTER: True
             },
+            'inheritance': {'mode': COMPOUND_HET},
             'annotations': {
                 'splice_ai': 0.5,
             },

seqr/models.py

@@ -862,7 +862,8 @@ def __unicode__(self):
     class Meta:
         unique_together = ('xpos', 'xpos_end', 'variant_id', 'family')
 
-        json_fields = ['guid', 'xpos', 'ref', 'alt', 'variant_id', 'selected_main_transcript_id', 'acmg_classification']
+        json_fields = ['guid', 'key', 'xpos', 'ref', 'alt', 'variant_id', 'selected_main_transcript_id', 'acmg_classification',
+                        'sv_type', 'gene_ids', 'genotypes']
 
 
 class VariantTagType(ModelWithGUID):

seqr/views/apis/report_api.py

@@ -943,10 +943,14 @@ def _add_row(row, family_id, row_type):
             for data_type, analysed in analysed_by_family_type[family_id].items()
         ]
         inheritance_models = f.pop('inheritance_models', [])
+        family_structure = _get_family_structure(
+            sum(1 for individual in individuals_by_id.values() if individual.get('date_data_generation')),
+            sum(1 for id in known_ids.values() if individuals_by_id.get(id, {}).get('date_data_generation')),
+        )
         f.update({
             'individual_count': len(individuals_by_id),
             'other_individual_ids':  '; '.join(sorted(individuals_ids)),
-            'family_structure': _get_family_structure(len(individuals_by_id), sum(1 for id in known_ids.values() if id)),
+            'family_structure': family_structure,
             'data_type': earliest_sample.get('data_type'),
             'date_data_generation': earliest_sample.get('date_data_generation'),
             'genes': '; '.join(sorted(f.get('genes', []))),

seqr/views/apis/report_api_tests.py

@@ -1294,6 +1294,28 @@ def test_family_metadata(self):
             'known_condition_name': '',
             'condition_inheritance': 'Unknown',
         })
+        test_no_data_row = next(r for r in response_json['rows'] if r['familyGuid'] == 'F000009_9')
+        self.assertDictEqual(test_no_data_row, {
+            'projectGuid': 'R0001_1kg',
+            'internal_project_id': '1kg project nåme with uniçøde',
+            'familyGuid': 'F000009_9',
+            'family_id': '9',
+            'displayName': '9',
+            'solve_status': 'Unsolved',
+            'actual_inheritance': '',
+            'date_data_generation': None,
+            'data_type': None,
+            'other_individual_ids': 'NA20878',
+            'individual_count': 1,
+            'family_structure': 'other',
+            'genes': '',
+            'pmid_id': None,
+            'phenotype_description': None,
+            'analysisStatus': 'Q',
+            'analysis_groups': '',
+            'analysed_by': '',
+            'consanguinity': 'Unknown',
+        })
 
         # Test empty project
         empty_project_url = reverse(family_metadata, args=['R0002_empty'])

seqr/views/apis/saved_variant_api.py

@@ -93,7 +93,7 @@ def create_manual_saved_variant_handler(request, family_guid):
     saved_variants = SavedVariant.objects.filter(family=family, guid__in=saved_variant_guids)
     _update_tags(saved_variants, {'tags': tags}, request.user)
 
-    response = get_json_for_saved_variants_with_tags(saved_variants, add_details=True, genome_version=genome_version)
+    response = get_json_for_saved_variants_with_tags(saved_variants)
     return create_json_response(response)
 
 
@@ -125,7 +125,7 @@ def create_saved_variant_handler(request):
     elif variant_json.get('tags'):
         _update_tags(saved_variants, variant_json, request.user)
 
-    response.update(get_json_for_saved_variants_with_tags(saved_variants, add_details=True, genome_version=family.project.genome_version))
+    response.update(get_json_for_saved_variants_with_tags(saved_variants))
     return create_json_response(response)