diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 56368fb8ef..fb021f24f5 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.9] + python-version: [3.9, 3.11] services: # Label used to access the service container diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 7351b83809..8f78c87766 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -1117,11 +1117,7 @@ def search_padded_interval(self, chrom, pos, padding): return self.filter(interval_q).result_values() @staticmethod - def _interval_query(chrom, start, end, offset=None, **kwargs): - if offset: - offset_pos = int((end - start) * offset) - start = max(start - offset_pos, MIN_POS) - end = min(end + offset_pos, MAX_POS) + def _interval_query(chrom, start, end, **kwargs): return Q(xpos__range=(get_xpos(chrom, start), get_xpos(chrom, end))) def _filter_seqr_frequency(self, entries, ac=None, hh=None, **kwargs): diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index fd8106fea8..061d2c63b0 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -189,15 +189,6 @@ def _get_comp_het_results_queryset(annotations_cls, primary_q, secondary_q, num_ {2: ('arrayIntersect(primary_familyGuids, secondary_familyGuids)', 'has({value}, {field})')}, ]), } - elif results.has_annotation('secondary_genotypes'): - genotype_expressions = { - 'primary_familyGenotypes': ArrayFilter('primary_familyGenotypes', conditions=[ - {1: ('secondary_familyGuids', 'has({value}, {field})')}, - ]), - 'secondary_genotypes': ArrayFilter('secondary_genotypes', conditions=[ - {2: (None, 'arrayExists(g -> g.1 = {field}, primary_familyGenotypes)')}, - ]), - } else: genotype_expressions = { 'primary_familyGenotypes': ArrayFilter('primary_familyGenotypes', conditions=[ @@ -699,10 +690,11 @@ def delete_clickhouse_family(project, family_guid, dataset_type, sample_type=Non return f'Clickhouse does not support deleting individual families from project. Manually delete {dataset_type} data for {family_guid} in project {project.guid}' +SV_DATASET_TYPES = { + Sample.SAMPLE_TYPE_WGS: Sample.DATASET_TYPE_SV_CALLS, + Sample.SAMPLE_TYPE_WES: 'GCNV', +} def _clickhouse_dataset_type(dataset_type, sample_type): if dataset_type == Sample.DATASET_TYPE_SV_CALLS: - if not sample_type: - raise ValueError('sample_type is required when dataset_type is SV') - if sample_type == Sample.SAMPLE_TYPE_WES: - return 'GCNV' + dataset_type = SV_DATASET_TYPES[sample_type] return dataset_type \ No newline at end of file diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 8aa7186072..5869c0924d 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -599,7 +599,7 @@ def test_location_search(self): [SV_VARIANT1, SV_VARIANT2], locus=sv_locus, ) self._assert_expected_search( - [SV_VARIANT1], locus={'rawItems': 'chr1:9292894-9369532'} + [SV_VARIANT1], locus={'rawItems': 'chr1:9297894-9369732%10'} ) self._reset_search_families() diff --git a/reference_data/management/commands/utils/__init__.py b/reference_data/management/commands/utils/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/reference_data/management/commands/utils/update_utils.py b/reference_data/management/commands/utils/update_utils.py deleted file mode 100644 index c94a654ca1..0000000000 --- a/reference_data/management/commands/utils/update_utils.py +++ /dev/null @@ -1,26 +0,0 @@ -from django.core.management.base import BaseCommand - -from reference_data.utils.gene_utils import get_genes_by_id_and_symbol -from reference_data.models import GeneMetadataModel - - -class ReferenceDataHandler(object): - - model_cls = GeneMetadataModel - - def __init__(self, **kwargs): - gene_ids_to_gene, gene_symbols_to_gene = get_genes_by_id_and_symbol() - self.gene_reference = { - 'gene_symbols_to_gene': gene_symbols_to_gene, - 'gene_ids_to_gene': gene_ids_to_gene, - } - - def update_records(self, **kwargs): - self.model_cls.update_records(**self.gene_reference, **kwargs) - - -class GeneCommand(BaseCommand): - reference_data_handler = ReferenceDataHandler - - def handle(self, *args, **options): - self.reference_data_handler(**options).update_records() diff --git a/requirements-dev.in b/requirements-dev.in index f85a139ed5..b44a43aea8 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,5 +1,5 @@ -c requirements.txt # use the generated reqs as a constraint -coverage<5.2 +coverage<6.6 django-compressor django-debug-toolbar # https://github.com/jazzband/django-debug-toolbar mock # mock objects for unit tests diff --git a/requirements-dev.txt b/requirements-dev.txt index 912bef8526..63a9e63bbe 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -20,7 +20,7 @@ charset-normalizer==3.0.1 # requests click==8.1.3 # via pip-tools -coverage==5.1 +coverage==6.5 # via -r requirements-dev.in django==4.2.24 # via diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 27d38ed371..9111e6baf7 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -36,7 +36,6 @@ 'GCNV': f'{Sample.DATASET_TYPE_SV_CALLS}_WES', Sample.DATASET_TYPE_SV_CALLS: f'{Sample.DATASET_TYPE_SV_CALLS}_WGS', } -USER_EMAIL = 'manage_command' RELATEDNESS_CHECK_NAME = 'relatedness_check' PDO_COPY_FIELDS = [ @@ -255,7 +254,7 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers # Reload saved variant JSON update_projects_saved_variant_json([ (project.id, project.guid, project.name, project.genome_version, families) for project, families in families_by_project.items() - ], user_email=USER_EMAIL, dataset_type=dataset_type, update_function=cls._update_project_saved_variant_genotypes, samples=updated_samples, clickhouse_dataset_type=clickhouse_dataset_type) + ], dataset_type=dataset_type, update_function=cls._update_project_saved_variant_genotypes, samples=updated_samples, clickhouse_dataset_type=clickhouse_dataset_type) @classmethod def _is_internal_project(cls, project): @@ -390,7 +389,7 @@ def _update_individuals_sample_qc(cls, sample_type, family_guids, sample_qc_map) ) @classmethod - def _update_project_saved_variant_genotypes(cls, project_id, genome_version, user_email, family_guids, project_guid, samples=None, clickhouse_dataset_type=None, **kwargs): + def _update_project_saved_variant_genotypes(cls, project_id, genome_version, family_guids, project_guid, samples=None, clickhouse_dataset_type=None, **kwargs): updates = {} for family_guid in family_guids: variant_models_by_key = { diff --git a/seqr/management/commands/reload_saved_variant_json.py b/seqr/management/commands/reload_saved_variant_json.py index 6d64f991e7..ce3e0edbab 100644 --- a/seqr/management/commands/reload_saved_variant_json.py +++ b/seqr/management/commands/reload_saved_variant_json.py @@ -30,5 +30,5 @@ def handle(self, *args, **options): family_ids = [family_guid] if family_guid else None project_list = [(*project, family_ids) for project in projects.order_by('id').values_list('id', 'guid', 'name', 'genome_version')] - update_projects_saved_variant_json(project_list, user_email='manage_command', update_function=update_project_saved_variant_json) + update_projects_saved_variant_json(project_list, update_function=update_project_saved_variant_json) logger.info("Done") diff --git a/seqr/utils/communication_utils.py b/seqr/utils/communication_utils.py index 7a57cfc173..128b73dd29 100644 --- a/seqr/utils/communication_utils.py +++ b/seqr/utils/communication_utils.py @@ -25,10 +25,9 @@ def _post_to_slack(channel, message): return None slack = Slacker(SLACK_TOKEN) - response = slack.chat.post_message( + slack.chat.post_message( channel, message, as_user=False, icon_emoji=":beaker:", username="Beaker (engineering-minion)", ) - return response.raw def send_welcome_email(user, referrer): diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py index e6be5cc0a2..a4bfc081ef 100644 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ b/seqr/utils/search/elasticsearch/es_utils_tests.py @@ -39,6 +39,11 @@ def replace_json(self, url, *args, **kwargs): def call_request_json(self, index=-1): return json.loads(self.calls[index].request.body) + def activate(self, func): + def wrapper(*args, **kwargs): + with self: + return func(*args, **kwargs) + return wrapper urllib3_responses = Urllib3Responses() diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 2122c99e7c..02d0c5508f 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -560,14 +560,6 @@ def _filter_inheritance_family_samples(samples, inheritance_filter): ] -def _format_interval(chrom=None, start=None, end=None, offset=None, **kwargs): - if offset: - offset_pos = int((end - start) * offset) - start = max(start - offset_pos, MIN_POS) - end = min(end + offset_pos, MAX_POS) - return [chrom, start, end] - - LIFTOVERS = { GENOME_VERSION_GRCh38: None, GENOME_VERSION_GRCh37: None, diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 035be59d6d..3bccba7935 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1274,9 +1274,6 @@ def _test_expected_vcf_responses(self, response, url): response = self.client.get(url, content_type='application/json') self.assertEqual(response.status_code, 200) - def _assert_expected_read_vcf_header_subprocess_calls(self, body): - return True - def test_validate_callset(self): url = reverse(validate_callset) self.check_pm_login(url) diff --git a/seqr/views/apis/dataset_api_tests.py b/seqr/views/apis/dataset_api_tests.py index 091f3abc08..af8b8a2fe2 100644 --- a/seqr/views/apis/dataset_api_tests.py +++ b/seqr/views/apis/dataset_api_tests.py @@ -451,12 +451,6 @@ class LocalDatasetAPITest(AuthenticationTestCase, DatasetAPITest): fixtures = ['users', '1kg_project'] -def assert_no_anvil_calls(self): - self.mock_list_workspaces.assert_not_called() - self.mock_get_ws_access_level.assert_not_called() - self.assert_no_extra_anvil_calls() - - # Test for permissions from AnVIL only class AnvilDatasetAPITest(AnvilAuthenticationTestCase, DatasetAPITest): fixtures = ['users', 'social_auth', '1kg_project'] diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index b896e2368e..81d8c13f87 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -390,7 +390,7 @@ def test_project_families(self): gene_ids = self._assert_expected_project_families(url, response_keys) self.assertSetEqual(gene_ids, {'ENSG00000135953', 'ENSG00000240361'}) - def _assert_expected_project_families(self, url, response_keys): + def _assert_expected_project_families(self, url, response_keys, no_discovery_tags=False): response = self.client.get(url) self.assertEqual(response.status_code, 200) @@ -436,12 +436,14 @@ def _assert_expected_project_families(self, url, response_keys): self.assertListEqual(family_3['discoveryTags'], []) self.assertListEqual(empty_family['discoveryTags'], []) - self.assertListEqual(family_1['discoveryTags'], [{ + family_1_tags = [] if no_discovery_tags else [{ 'transcripts': {'ENSG00000135953': [mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY]}, 'mainTranscriptId': 'ENST00000258436', 'selectedMainTranscriptId': None, - }]) - self.assertListEqual(response_json['familiesByGuid']['F000002_2']['discoveryTags'], [self.DISCOVERY_TAG]) + }] + self.assertListEqual(family_1['discoveryTags'], family_1_tags) + family_2_tags = [] if no_discovery_tags else [self.DISCOVERY_TAG] + self.assertListEqual(response_json['familiesByGuid']['F000002_2']['discoveryTags'], family_2_tags) no_discovery_families = set(response_json['familiesByGuid'].keys()) - {'F000001_1', 'F000002_2'} self.assertSetEqual({ len(response_json['familiesByGuid'][family_guid]['discoveryTags']) for family_guid in no_discovery_families @@ -802,8 +804,8 @@ def _assert_expected_project_families(self, *args, **kwargs): self.reset_logs() connections['clickhouse'].close() self.DISCOVERY_TAG = {**DISCOVERY_TAG, 'transcripts': {}} - no_clickhouse_gene_ids = super()._assert_expected_project_families(*args, **kwargs) - self.assertSetEqual(no_clickhouse_gene_ids, {'ENSG00000135953'}) + no_clickhouse_gene_ids = super()._assert_expected_project_families(*args, **kwargs, no_discovery_tags=True) + self.assertSetEqual(no_clickhouse_gene_ids, set()) self.assert_json_logs(None, [ ("Error loading discovery genes from clickhouse: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.", { 'severity': 'ERROR', diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index e637e90bad..a434bddca0 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -327,20 +327,17 @@ def _assert_expected_rnaseq_response(self, response_json): 'start': 132885746, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}] ) - def _assert_expected_results_family_context(self, response_json, locus_list_detail=False, skip_gene_context=False): - if not skip_gene_context: - self._assert_expected_results_context(response_json, locus_list_detail=locus_list_detail) + def _assert_expected_results_family_context(self, response_json, locus_list_detail=False): + self._assert_expected_results_context(response_json, locus_list_detail=locus_list_detail) family_fields = {'individualGuids'} family_fields.update(FAMILY_FIELDS) if len(response_json['familiesByGuid']) > 1: self.assertSetEqual(set(response_json['familiesByGuid']['F000002_2'].keys()), family_fields) - if not skip_gene_context: - family_fields.add('tpmGenes') + family_fields.add('tpmGenes') self.assertSetEqual(set(response_json['familiesByGuid']['F000001_1'].keys()), family_fields) - if not skip_gene_context: - self.assertSetEqual(set(response_json['familiesByGuid']['F000001_1']['tpmGenes']), {'ENSG00000227232'}) + self.assertSetEqual(set(response_json['familiesByGuid']['F000001_1']['tpmGenes']), {'ENSG00000227232'}) self.assertEqual(len(response_json['individualsByGuid']), len(response_json['familiesByGuid'])*3) individual_fields = {'igvSampleGuids'} @@ -353,8 +350,7 @@ def _assert_expected_results_family_context(self, response_json, locus_list_deta self.assertEqual(len(response_json['familyNotesByGuid']), 3) self.assertSetEqual(set(response_json['familyNotesByGuid']['FAN000001_1'].keys()), FAMILY_NOTE_FIELDS) - if not skip_gene_context: - self._assert_expected_rnaseq_response(response_json) + self._assert_expected_rnaseq_response(response_json) def _assert_expected_results_context(self, response_json, has_pa_detail=True, locus_list_detail=False, rnaseq=True): gene_fields = {'locusListGuids'} @@ -869,7 +865,7 @@ def test_query_single_variant(self, mock_get_variant): self.assertEqual(response.status_code, 400) self.assertEqual(response.json()['error'], 'Variant not found') - def _assert_expected_single_variant_results_context(self, response_json, omit_fields=None, no_metadata=False, **expected_response): + def _assert_expected_single_variant_results_context(self, response_json, omit_fields=None, **expected_response): omit_fields = {'search', *(omit_fields or [])} expected_search_response = deepcopy({**self.EXPECTED_SEARCH_RESPONSE, **EXPECTED_SEARCH_FAMILY_CONTEXT}) @@ -879,25 +875,18 @@ def _assert_expected_single_variant_results_context(self, response_json, omit_fi }) for k in omit_fields: expected_search_response.pop(k) - if no_metadata: - expected_search_response.update({k: {} for k in { - 'savedVariantsByGuid', 'variantTagsByGuid', 'variantFunctionalDataByGuid', 'genesById', - 'rnaSeqData', 'phenotypeGeneScores', 'mmeSubmissionsByGuid' - }}) - expected_search_response.pop('transcriptsById', None) - else: - expected_search_response['savedVariantsByGuid'].pop('SV0000002_1248367227_r0390_100') - expected_search_response['variantTagsByGuid'] = { - k: EXPECTED_SEARCH_RESPONSE['variantTagsByGuid'][k] - for k in {'VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100'} - } + expected_search_response['savedVariantsByGuid'].pop('SV0000002_1248367227_r0390_100') + expected_search_response['variantTagsByGuid'] = { + k: EXPECTED_SEARCH_RESPONSE['variantTagsByGuid'][k] + for k in {'VT1708633_2103343353_r0390_100', 'VT1726961_2103343353_r0390_100'} + } expected_search_response['variantNotesByGuid'] = {} expected_search_response['genesById'] = { k: v for k, v in expected_search_response['genesById'].items() if k in {'ENSG00000227232', 'ENSG00000268903'} } self.assertSetEqual(set(response_json.keys()), set(expected_search_response.keys())) self.assertDictEqual(response_json, expected_search_response) - self._assert_expected_results_family_context(response_json, locus_list_detail=True, skip_gene_context=no_metadata) + self._assert_expected_results_family_context(response_json, locus_list_detail=True) self.assertSetEqual(set(response_json['projectsByGuid'][PROJECT_GUID].keys()), PROJECT_TAG_TYPE_FIELDS) self.assertSetEqual(set(response_json['familiesByGuid'].keys()), {'F000001_1'}) diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index 82121c696f..546aa2fb2c 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -1,9 +1,11 @@ from collections import defaultdict from django.db.models import Count, Q, F, prefetch_related_objects +from django.contrib.postgres.aggregates import ArrayAgg +from django.db.models.functions import JSONObject -from clickhouse_search.search import get_transcripts_by_key +from clickhouse_search.search import get_transcripts_by_key, get_annotations_queryset from seqr.models import Individual, IgvSample, AnalysisGroup, DynamicAnalysisGroup, LocusList, VariantTagType,\ - VariantFunctionalData, FamilyNote, SavedVariant, VariantTag, VariantNote + VariantFunctionalData, FamilyNote, SavedVariant, VariantTag, VariantNote, Sample from seqr.utils.gene_utils import get_genes from seqr.utils.logging_utils import SeqrLogger from seqr.utils.search.utils import backend_specific_call @@ -119,22 +121,22 @@ def families_discovery_tags(families, genome_version, project=None): families_by_guid = {f['familyGuid']: dict(discoveryTags=[], **f) for f in families} family_filter = {'family__project': project} if project else {'family__guid__in': families_by_guid.keys()} - discovery_tags_by_key = {v.pop('key'): v for v in SavedVariant.objects.filter( + discovery_variants = SavedVariant.objects.filter( varianttag__variant_tag_type__category='CMG Discovery Tags', **family_filter, - ).values( - 'key', 'family__guid', selectedMainTranscriptId=F('selected_main_transcript_id'), - transcripts=F('saved_variant_json__transcripts'), mainTranscriptId=F('saved_variant_json__mainTranscriptId'), - )} + ) try: - backend_specific_call(lambda *args: None, _add_clickhouse_transcripts)(discovery_tags_by_key, genome_version) + discovery_tags = backend_specific_call(_get_no_key_tags, _get_clickhouse_tags)( + discovery_variants, genome_version=genome_version, + ) except Exception as e: logger.error(f'Error loading discovery genes from clickhouse: {e}', None) + discovery_tags = [] gene_ids = set() - for tag in discovery_tags_by_key.values(): + for tag in discovery_tags: tag['transcripts'] = tag.get('transcripts') or {} gene_ids.update(list(tag['transcripts'].keys())) - families_by_guid[tag.pop('family__guid')]['discoveryTags'].append(tag) + families_by_guid[tag.pop('family_guid')]['discoveryTags'].append(tag) return { 'familiesByGuid': families_by_guid, @@ -142,15 +144,37 @@ def families_discovery_tags(families, genome_version, project=None): } -def _add_clickhouse_transcripts(discovery_tags_by_key, genome_version): - transcripts_by_key = get_transcripts_by_key(genome_version, discovery_tags_by_key.keys()) - for key, tag in discovery_tags_by_key.items(): - if key in transcripts_by_key: +def _get_no_key_tags(discovery_variants, **kwargs): + return discovery_variants.values( + family_guid=F('family__guid'), selectedMainTranscriptId=F('selected_main_transcript_id'), + transcripts=F('saved_variant_json__transcripts'), mainTranscriptId=F('saved_variant_json__mainTranscriptId'), + ) + + +def _get_clickhouse_tags(discovery_variants, genome_version): + discovery_tags = list(_get_no_key_tags(discovery_variants.filter(key__isnull=True))) + + tags_by_dataset_type = discovery_variants.filter(key__isnull=False).values('dataset_type').annotate( + keys=ArrayAgg('key', distinct=True), + tags=ArrayAgg(JSONObject(key='key', family_guid='family__guid', selectedMainTranscriptId='selected_main_transcript_id')), + ) + + for dataset_type, keys, tags in tags_by_dataset_type.values_list('dataset_type', 'keys', 'tags'): + if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: + transcripts_by_key = get_transcripts_by_key(genome_version, keys) + else: + qs = get_annotations_queryset(genome_version, dataset_type, keys) + transcripts_by_key = dict(qs.values_list('key', qs.transcript_field)) + for tag in tags: + key = tag.pop('key') tag['transcripts'] = transcripts_by_key[key] - tag['mainTranscriptId'] = next( + tag['mainTranscriptId'] = next(( t['transcriptId'] for gene_transcripts in tag['transcripts'].values() for t in gene_transcripts - if t['transcriptRank'] == 0 - ) + if t.get('transcriptRank') == 0 + ), None) + discovery_tags.append(tag) + + return discovery_tags MME_TAG_NAME = 'MME Submission' diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index a1663e3f65..1371b142de 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -35,7 +35,7 @@ OMIM_GENOME_VERSION = GENOME_VERSION_GRCh38 -def update_projects_saved_variant_json(projects, user_email, update_function, **kwargs): +def update_projects_saved_variant_json(projects, update_function, **kwargs): success = {} skipped = {} error = {} @@ -43,7 +43,7 @@ def update_projects_saved_variant_json(projects, user_email, update_function, ** for project_id, project_guid, project_name, genome_version, family_guids in tqdm(projects, unit=' project'): try: updated_saved_variants = update_function( - project_id, genome_version, user_email=user_email, family_guids=family_guids, project_guid=project_guid, **kwargs) + project_id, genome_version, family_guids=family_guids, project_guid=project_guid, **kwargs) if updated_saved_variants is None: skipped[project_name] = True else: @@ -68,7 +68,7 @@ def update_projects_saved_variant_json(projects, user_email, update_function, ** logger.info(f' {k}: {v}') -def get_saved_variants(genome_version, project_id=None, family_guids=None, dataset_type=None, clickhouse_dataset_type=None): +def get_saved_variants(genome_version, project_id=None, family_guids=None, clickhouse_dataset_type=None): saved_variants = SavedVariant.objects.filter( Q(saved_variant_json__genomeVersion__isnull=True) | Q(saved_variant_json__genomeVersion=genome_version.replace('GRCh', '')) @@ -77,24 +77,11 @@ def get_saved_variants(genome_version, project_id=None, family_guids=None, datas saved_variants = saved_variants.filter(family__project_id=project_id) if family_guids: saved_variants = saved_variants.filter(family__guid__in=family_guids) - if dataset_type: - saved_variants = saved_variants.filter(**saved_variants_dataset_type_filter(dataset_type)) - elif clickhouse_dataset_type: + if clickhouse_dataset_type: saved_variants = saved_variants.filter(dataset_type=clickhouse_dataset_type) return saved_variants -def saved_variants_dataset_type_filter(dataset_type): - xpos_filter_key = 'xpos__gte' if dataset_type == Sample.DATASET_TYPE_MITO_CALLS else 'xpos__lt' - dataset_filter = {xpos_filter_key: get_xpos('M', 1)} - if dataset_type == Sample.DATASET_TYPE_SV_CALLS: - dataset_filter['alt__isnull'] = True - else: - # Filter out manual variants with invalid characters, such as those used for STRs - dataset_filter['alt__regex'] = '^[ACGT]+$' - return dataset_filter - - def parse_saved_variant_json(variant_json, family_id, variant_id=None,): if 'xpos' not in variant_json: variant_json['xpos'] = get_xpos(variant_json['chrom'], variant_json['pos'])