From 5641d2039f238496bbd83ae1305f26f638e5aaab Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 12:25:47 -0400 Subject: [PATCH 01/89] remove hail from backed specific call --- clickhouse_search/search.py | 4 --- seqr/utils/search/utils.py | 54 +++++++++++++------------------------ 2 files changed, 18 insertions(+), 40 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 678b554184..e9de9ab1f8 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -29,10 +29,6 @@ SELECTED_TRANSCRIPT_FIELD = 'selectedTranscript' -def clickhouse_backend_enabled(): - return bool(CLICKHOUSE_SERVICE_HOSTNAME) - - def get_clickhouse_variants(samples, search, user, previous_search_results, genome_version,page=1, num_results=100, sort=None, **kwargs): sample_data_by_dataset_type = _get_sample_data(samples) results = [] diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 355b098eef..c35eb6a1ed 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -3,7 +3,7 @@ from datetime import timedelta from django.db.models import Count -from clickhouse_search.search import clickhouse_backend_enabled, get_clickhouse_variants, format_clickhouse_results, \ +from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, \ get_clickhouse_cache_results, clickhouse_variant_lookup, get_clickhouse_variant_by_id from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 from seqr.models import Sample, Individual, Project @@ -14,8 +14,6 @@ from seqr.utils.search.elasticsearch.es_utils import ping_elasticsearch, delete_es_index, get_elasticsearch_status, \ get_es_variants, get_es_variants_for_variant_ids, process_es_previously_loaded_results, process_es_previously_loaded_gene_aggs, \ es_backend_enabled, ping_kibana, ES_EXCEPTION_ERROR_MAP, ES_EXCEPTION_MESSAGE_MAP, ES_ERROR_LOG_EXCEPTIONS -from seqr.utils.search.hail_search_utils import get_hail_variants, get_hail_variants_for_variant_ids, ping_hail_backend, \ - hail_variant_lookup from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.xpos_utils import get_xpos, format_chrom, MIN_POS, MAX_POS @@ -59,26 +57,24 @@ def _raise_clickhouse_not_implemented(*args, **kwargs): raise NotImplementedError('Clickhouse backend is not implemented for this function.') -def backend_specific_call(es_func, hail_backend_func, clickhouse_func): +def backend_specific_call(es_func, clickhouse_func): if es_backend_enabled(): return es_func - elif clickhouse_backend_enabled(): - return clickhouse_func else: - return hail_backend_func + return clickhouse_func def ping_search_backend(): # Clickhouse backend does not need special uptime testing, will be checked with the other database connection pings - backend_specific_call(ping_elasticsearch, ping_hail_backend, lambda: None)() + backend_specific_call(ping_elasticsearch, lambda: None)() def ping_search_backend_admin(): - backend_specific_call(ping_kibana, lambda: True, lambda: True)() + backend_specific_call(ping_kibana, lambda: True)() def get_search_backend_status(): - return backend_specific_call(get_elasticsearch_status, _raise_search_error('Elasticsearch is disabled'), _raise_search_error('Elasticsearch is disabled'))() + return backend_specific_call(get_elasticsearch_status, _raise_search_error('Elasticsearch is disabled'))() def _get_filtered_search_samples(search_filter, active_only=True): @@ -129,7 +125,6 @@ def delete_search_backend_data(data_id): return backend_specific_call( delete_es_index, _raise_search_error('Deleting indices is disabled without the elasticsearch backend'), - _raise_search_error('Deleting indices is disabled without the elasticsearch backend'), )(data_id) @@ -139,7 +134,6 @@ def get_single_variant(family, variant_id, user=None): samples = _get_families_search_data([family], dataset_type, sample_filter={'individual__family_id': family.id}) variant = backend_specific_call( _process_ids_search(get_es_variants_for_variant_ids), - _process_ids_search(get_hail_variants_for_variant_ids), _get_clickhouse_variant_by_id, )(parsed_variant_id, variant_id, samples, family.project.genome_version, dataset_type, user) if not variant: @@ -173,7 +167,7 @@ def get_variants_for_variant_ids(families, variant_ids, dataset_type=None, user= } dataset_type = _variant_ids_dataset_type(parsed_variant_ids.values()) - return backend_specific_call(get_es_variants_for_variant_ids, get_hail_variants_for_variant_ids, _raise_clickhouse_not_implemented)( + return backend_specific_call(get_es_variants_for_variant_ids, _raise_clickhouse_not_implemented)( _get_families_search_data(families, dataset_type=dataset_type), _get_search_genome_version(families), parsed_variant_ids, user, user_email=user_email, ) @@ -199,7 +193,7 @@ def _validate_dataset_type_genome_version(dataset_type, genome_version): def variant_lookup(user, parsed_variant_id, **kwargs): dataset_type = DATASET_TYPES_LOOKUP[_variant_ids_dataset_type([parsed_variant_id])][0] - lookup_func = backend_specific_call(_raise_search_error('Lookup is disabled'), hail_variant_lookup, clickhouse_variant_lookup) + lookup_func = backend_specific_call(_raise_search_error('Lookup is disabled'), clickhouse_variant_lookup) return _variant_lookup(lookup_func, user, parsed_variant_id, **kwargs, dataset_type=dataset_type) @@ -220,7 +214,7 @@ def _sv_variant_lookup(user, variant_id, dataset_type, samples, genome_version=N data_type = f'{dataset_type}_{sample_type}' lookup_samples = samples.filter(sample_type=sample_type) - lookup_func = backend_specific_call(_raise_search_error('Lookup is disabled'), hail_variant_lookup, clickhouse_variant_lookup) + lookup_func = backend_specific_call(_raise_search_error('Lookup is disabled'), clickhouse_variant_lookup) variant = lookup_func(user, variant_id, data_type, samples=lookup_samples, genome_version=genome_version, **kwargs) variants = [variant] @@ -258,7 +252,7 @@ def _get_cached_search_results(search_model, sort=None, family_guid=None): results = safe_redis_get_json(cache_key) if not results: results = backend_specific_call( - lambda *args: None, lambda *args: None, _process_clickhouse_unsorted_cached_results, + lambda *args: None, _process_clickhouse_unsorted_cached_results, )(cache_key, sort, family_guid) return results or {} @@ -284,14 +278,13 @@ def query_variants(search_model, sort=XPOS_SORT_KEY, skip_genotype_filter=False, loaded_results = previous_search_results.get('all_results') or [] if len(loaded_results) >= end_index: results_page = backend_specific_call( - lambda results, genome_version: results, lambda results, genome_version: results, format_clickhouse_results, + lambda results, genome_version: results, format_clickhouse_results, )(loaded_results[start_index:end_index], genome_version) return results_page, total_results previously_loaded_results = backend_specific_call( process_es_previously_loaded_results, lambda *args: None, # Other backends need no additional parsing - lambda *args: None, )(previous_search_results, start_index, end_index) if previously_loaded_results is not None: return previously_loaded_results, total_results @@ -345,7 +338,7 @@ def _query_variants(search_model, user, previous_search_results, genome_version, parsed_search = { 'parsed_locus': backend_specific_call( - lambda genome_version, **kwargs: kwargs, _parse_locus_intervals, _parse_locus_gene_intervals, + lambda genome_version, **kwargs: kwargs, _parse_locus_gene_intervals, )(genome_version, genes=genes, intervals=intervals, rs_ids=rs_ids, variant_ids=variant_ids, parsed_variant_ids=parsed_variant_ids, exclude_locations=exclude_locations), } @@ -381,7 +374,7 @@ def _query_variants(search_model, user, previous_search_results, genome_version, def _execute_search(samples, parsed_search, user, previous_search_results, genome_version, **kwargs): - return backend_specific_call(get_es_variants, get_hail_variants, get_clickhouse_variants)( + return backend_specific_call(get_es_variants, get_clickhouse_variants)( samples, parsed_search, user, previous_search_results, genome_version, **kwargs, ) @@ -397,7 +390,6 @@ def get_variant_query_gene_counts(search_model, user): previously_loaded_results = backend_specific_call( process_es_previously_loaded_gene_aggs, lambda *args: None, # Other backends need no additional parsing - lambda *args: None, )(previous_search_results) if previously_loaded_results is not None: return previously_loaded_results @@ -405,27 +397,25 @@ def get_variant_query_gene_counts(search_model, user): genome_version = _get_search_genome_version(search_model.families.all()) gene_counts, _ = _query_variants(search_model, user, previous_search_results, genome_version, gene_agg=True) gene_counts = backend_specific_call( - lambda *args: None, lambda *args: None, _get_gene_aggs_for_cached_variants, + lambda *args: None, _get_gene_aggs_for_cached_variants, )(previous_search_results) or gene_counts return gene_counts def _get_gene_aggs_for_cached_variants(previous_search_results): gene_aggs = defaultdict(lambda: {'total': 0, 'families': defaultdict(int)}) - # ES caches compound hets separately from main results, hail search caches everything together + # ES caches compound hets separately from main results, other backends cache everything together flattened_variants = backend_specific_call( lambda results: results, lambda results: [v for variants in results for v in (variants if isinstance(variants, list) else [variants])], - lambda results: [v for variants in results for v in (variants if isinstance(variants, list) else [variants])], )(previous_search_results['all_results']) for var in flattened_variants: - # ES only reports breakdown for main transcript gene only, hail backend reports for all genes + # ES only reports breakdown for main transcript gene only, other backends report for all genes gene_ids = backend_specific_call( lambda variant: next(( [gene_id] for gene_id, transcripts in variant['transcripts'].items() if any(t['transcriptId'] == var['mainTranscriptId'] for t in transcripts) ), []) if var['mainTranscriptId'] else [], - lambda variant: variant['transcripts'].keys(), lambda variant: variant['transcripts'].keys() if 'transcripts' in variant else {t['geneId'] for t in variant['sortedTranscriptConsequences']}, )(var) for gene_id in gene_ids: @@ -576,7 +566,7 @@ def _validate_search(search, samples, previous_search_results): ) if not has_location_filter: - backend_specific_call(lambda *args: None, _validate_no_location_search, _validate_no_location_search)(samples) + backend_specific_call(lambda *args: None, _validate_no_location_search)(samples) MAX_FAMILY_COUNTS = {Sample.SAMPLE_TYPE_WES: 200, Sample.SAMPLE_TYPE_WGS: 35} @@ -595,7 +585,7 @@ def _validate_no_location_search(samples): def _filter_inheritance_family_samples(samples, inheritance_filter): family_groups = defaultdict(set) - sample_group_field = backend_specific_call('elasticsearch_index', 'dataset_type', 'dataset_type') + sample_group_field = backend_specific_call('elasticsearch_index', 'dataset_type') individual_affected_status = inheritance_filter.get('affected') or {} genotype_filter = None if inheritance_filter.get(Individual.AFFECTED_STATUS_AFFECTED) else inheritance_filter.get('genotype') for sample in samples: @@ -642,14 +632,6 @@ def _parse_locus_gene_intervals(genome_version, genes=None, intervals=None, rs_i } -def _parse_locus_intervals(*args, **kwargs): - parsed_locus = _parse_locus_gene_intervals(*args, **kwargs) - gene_intervals = parsed_locus.pop('gene_intervals') - if gene_intervals: - parsed_locus['intervals'] = (parsed_locus['intervals'] or []) + sorted(gene_intervals.values()) - return parsed_locus - - def _format_interval(chrom=None, start=None, end=None, offset=None, **kwargs): if offset: offset_pos = int((end - start) * offset) From 7506781420f6c18779b993b17c3cacbc529dc2ee Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 12:41:11 -0400 Subject: [PATCH 02/89] remove hail only command --- .../reload_saved_variant_annotations.py | 27 ---- .../reload_saved_variant_annotations_tests.py | 131 ------------------ 2 files changed, 158 deletions(-) delete mode 100644 seqr/management/commands/reload_saved_variant_annotations.py delete mode 100644 seqr/management/tests/reload_saved_variant_annotations_tests.py diff --git a/seqr/management/commands/reload_saved_variant_annotations.py b/seqr/management/commands/reload_saved_variant_annotations.py deleted file mode 100644 index a2481e1731..0000000000 --- a/seqr/management/commands/reload_saved_variant_annotations.py +++ /dev/null @@ -1,27 +0,0 @@ -from django.core.management.base import BaseCommand, CommandError -from reference_data.models import GENOME_VERSION_LOOKUP -from seqr.models import Sample -from seqr.management.commands.check_for_new_samples_from_pipeline import reload_shared_variant_annotations -from seqr.utils.search.hail_search_utils import search_data_type -from seqr.utils.search.utils import backend_specific_call - -DATA_TYPE_CHOICES = { - search_data_type(dt, st) for dt in Sample.DATASET_TYPE_LOOKUP for st in [Sample.SAMPLE_TYPE_WGS, Sample.SAMPLE_TYPE_WES] -} - - -def _clickhouse_error(): - raise CommandError('Reloading variant annotations is not supported in clickhouse') - - -class Command(BaseCommand): - help = 'Reload shared variant annotations for all saved variants' - - def add_arguments(self, parser): - parser.add_argument('data_type', choices=sorted(DATA_TYPE_CHOICES)) - parser.add_argument('genome_version', choices=sorted(GENOME_VERSION_LOOKUP.values())) - parser.add_argument('chromosomes', nargs='*', help='Chromosome(s) to reload. If not specified, defaults to all chromosomes.') - - def handle(self, *args, **options): - backend_specific_call(lambda: True, lambda: True, _clickhouse_error)() - reload_shared_variant_annotations(options['data_type'], options['genome_version'], chromosomes=options['chromosomes']) diff --git a/seqr/management/tests/reload_saved_variant_annotations_tests.py b/seqr/management/tests/reload_saved_variant_annotations_tests.py deleted file mode 100644 index 9df44a79d6..0000000000 --- a/seqr/management/tests/reload_saved_variant_annotations_tests.py +++ /dev/null @@ -1,131 +0,0 @@ -from django.core.management import call_command -from django.core.management.base import CommandError -import json -import mock -import responses - -from seqr.views.utils.test_utils import AnvilAuthenticationTestCase -from seqr.models import Sample, SavedVariant - -MOCK_HAIL_HOST = 'test-hail-host' -MOCK_HAIL_ORIGIN = f'http://{MOCK_HAIL_HOST}' - - -@mock.patch('seqr.utils.search.hail_search_utils.HAIL_BACKEND_SERVICE_HOSTNAME', MOCK_HAIL_HOST) -class ReloadVariantAnnotationsTest(AnvilAuthenticationTestCase): - fixtures = ['users', '1kg_project'] - - CLICKHOUSE_HOSTNAME = None - - @responses.activate - def test_command(self): - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup', status=200, json={ - 'results': [ - {'variantId': '1-46859832-G-A', 'updated_new_field': 'updated_value', 'rsid': 'rs123'}, - {'variantId': '1-248367227-TC-T', 'updated_field': 'updated_value'}, - ], - }) - - # Test errors - with self.assertRaises(CommandError) as ce: - call_command('reload_saved_variant_annotations') - self.assertEqual(str(ce.exception), 'Error: the following arguments are required: data_type, genome_version, chromosomes') - - with self.assertRaises(CommandError) as ce: - call_command('reload_saved_variant_annotations', 'SV', 'GRCh37') - self.assertEqual(str(ce.exception), "Error: argument data_type: invalid choice: 'SV' (choose from 'MITO', 'SNV_INDEL', 'SV_WES', 'SV_WGS')") - - # Test success - self.reset_logs() - call_command('reload_saved_variant_annotations', 'SNV_INDEL', 'GRCh37') - - self.assert_json_logs(user=None, expected=[ - ('Reloading shared annotations for 2 SNV_INDEL GRCh37 saved variants in chromosome 1 (2 unique)', None), - ('Fetched 2 additional variants in chromosome 1', None), - ('update 2 SavedVariants', {'dbUpdate': { - 'dbEntity': 'SavedVariant', - 'entityIds': ['SV0000002_1248367227_r0390_100', 'SV0059956_11560662_f019313_1'], - 'updateFields': ['saved_variant_json'], - 'updateType': 'bulk_update'}, - }), - ] + [(f'No additional SNV_INDEL GRCh37 saved variants to update in chromosome {chrom}', None) for chrom in ['2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20']] + [ - ('Reloading shared annotations for 1 SNV_INDEL GRCh37 saved variants in chromosome 21 (1 unique)', None), - ('Fetched 2 additional variants in chromosome 21', None), - ] + [(f'No additional SNV_INDEL GRCh37 saved variants to update in chromosome {chrom}', None) for chrom in ['22', 'X', 'Y', 'M']]) - - self.assertEqual(len(responses.calls), 2) - for call in responses.calls: - self.assertEqual(call.request.url, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup') - self.assertEqual(call.request.headers.get('From'), 'manage_command') - self.assertDictEqual(json.loads(responses.calls[0].request.body), { - 'genome_version': 'GRCh37', - 'data_type': 'SNV_INDEL', - 'variant_ids': [['1', 46859832, 'G', 'A'], ['1', 248367227, 'TC', 'T']], - }) - self.assertDictEqual(json.loads(responses.calls[1].request.body), { - 'genome_version': 'GRCh37', - 'data_type': 'SNV_INDEL', - 'variant_ids': [['21', 3343353, 'GAGA', 'G']], - }) - - annotation_updated_json_1 = SavedVariant.objects.get(guid='SV0000002_1248367227_r0390_100').saved_variant_json - self.assertEqual(len(annotation_updated_json_1), 17) - self.assertListEqual(annotation_updated_json_1['familyGuids'], ['F000001_1']) - self.assertEqual(annotation_updated_json_1['updated_field'], 'updated_value') - - annotation_updated_json_2 = SavedVariant.objects.get(guid='SV0059956_11560662_f019313_1').saved_variant_json - self.assertEqual(len(annotation_updated_json_2), 16) - self.assertEqual(annotation_updated_json_2['updated_new_field'], 'updated_value') - self.assertEqual(annotation_updated_json_2['rsid'], 'rs123') - self.assertEqual(annotation_updated_json_2['mainTranscriptId'], 'ENST00000505820') - self.assertEqual(len(annotation_updated_json_2['genotypes']), 3) - - # Test chromosome subset - responses.calls.reset() - self.reset_logs() - call_command('reload_saved_variant_annotations', 'SNV_INDEL', 'GRCh37', '3', '21') - - self.assert_json_logs(user=None, expected=[(log, None) for log in [ - 'No additional SNV_INDEL GRCh37 saved variants to update in chromosome 3', - 'Reloading shared annotations for 1 SNV_INDEL GRCh37 saved variants in chromosome 21 (1 unique)', - 'Fetched 2 additional variants in chromosome 21', - ]]) - - self.assertEqual(len(responses.calls), 1) - self.assertDictEqual(json.loads(responses.calls[0].request.body), { - 'genome_version': 'GRCh37', - 'data_type': 'SNV_INDEL', - 'variant_ids': [['21', 3343353, 'GAGA', 'G']], - }) - - responses.calls.reset() - self.reset_logs() - call_command('reload_saved_variant_annotations', 'SNV_INDEL', 'GRCh37', '3', '6') - self.assert_json_logs(user=None, expected=[ - (f'No additional SNV_INDEL GRCh37 saved variants to update in chromosome {chrom}', None) - for chrom in [3, 6] - ]) - self.assertEqual(len(responses.calls), 0) - - # Test SVs - responses.calls.reset() - Sample.objects.filter(guid='S000147_na21234').update(individual_id=20) - call_command('reload_saved_variant_annotations', 'SV_WGS', 'GRCh37') - - self.assertEqual(len(responses.calls), 1) - self.assertDictEqual(json.loads(responses.calls[0].request.body), { - 'genome_version': 'GRCh37', - 'data_type': 'SV_WGS', - 'variant_ids': ['prefix_19107_DEL'], - }) - - -class ClickhouseReloadVariantAnnotationsTest(AnvilAuthenticationTestCase): - - fixtures = ['users', '1kg_project'] - - def test_command(self): - with self.assertRaises(CommandError) as ce: - call_command('reload_saved_variant_annotations', 'SNV_INDEL', 'GRCh37') - self.assertEqual(str(ce.exception), 'Reloading variant annotations is not supported in clickhouse') - From e4e0ea59dc128318b90f155454aedcc5c005d815 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 12:48:29 -0400 Subject: [PATCH 03/89] clean up check samples job --- .../check_for_new_samples_from_pipeline.py | 126 +----------------- seqr/utils/search/utils.py | 4 +- seqr/views/utils/variant_utils.py | 3 - 3 files changed, 7 insertions(+), 126 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 899bb6a475..f980c72026 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -8,14 +8,11 @@ import re from clickhouse_search.search import get_clickhouse_genotypes -from reference_data.models import GENOME_VERSION_LOOKUP, GENOME_VERSION_GRCh38 +from reference_data.models import GENOME_VERSION_LOOKUP from seqr.models import Family, Sample, SavedVariant, Project, Individual from seqr.utils.communication_utils import safe_post_to_slack, send_project_email from seqr.utils.file_utils import file_iter, list_files, is_google_bucket_file_path from seqr.utils.search.add_data_utils import notify_search_data_loaded, update_airtable_loading_tracking_status -from seqr.utils.search.utils import parse_valid_variant_id, backend_specific_call -from seqr.utils.search.hail_search_utils import hail_variant_multi_lookup, search_data_type -from seqr.utils.xpos_utils import get_xpos, CHROMOSOMES, MIN_POS, MAX_POS from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS from seqr.views.utils.dataset_utils import match_and_update_search_samples from seqr.views.utils.export_utils import write_multiple_files @@ -29,7 +26,6 @@ CLICKHOUSE_MIGRATION_SENTINEL = 'hail_search_to_clickhouse_migration' RUN_FILE_PATH_TEMPLATE = '{data_dir}/{genome_version}/{dataset_type}/runs/{run_version}/{file_name}' -SUCCESS_FILE_NAME = '_SUCCESS' CLICKHOUSE_SUCCESS_FILE_NAME = '_CLICKHOUSE_LOAD_SUCCESS' VALIDATION_ERRORS_FILE_NAME = 'validation_errors.json' ERRORS_REPORTED_FILE_NAME = '_ERRORS_REPORTED' @@ -69,7 +65,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): runs = self._get_runs(**options) - success_file_name = backend_specific_call(None, SUCCESS_FILE_NAME, CLICKHOUSE_SUCCESS_FILE_NAME) + success_file_name = CLICKHOUSE_SUCCESS_FILE_NAME success_run_dirs = [run_dir for run_dir, run_details in runs.items() if success_file_name in run_details['files']] if success_run_dirs: self._load_success_runs(runs, success_run_dirs) @@ -95,38 +91,19 @@ def _load_success_runs(self, runs, success_run_dirs): return logger.info(f'Loading new samples from {len(success_run_dirs)} run(s)') - updated_families_by_data_type = defaultdict(set) - updated_variants_by_data_type = defaultdict(dict) for run_dir, run_details in new_runs.items(): try: if CLICKHOUSE_MIGRATION_SENTINEL in run_details["run_version"]: logging.info(f'Skipping ClickHouse migration {run_details["genome_version"]}/{run_details["dataset_type"]}: {run_details["run_version"]}') continue metadata_path = os.path.join(run_dir, 'metadata.json') - data_type, updated_families, updated_variants_by_id = self._load_new_samples(metadata_path, **run_details) - data_type_key = (data_type, run_details['genome_version']) - updated_families_by_data_type[data_type_key].update(updated_families) - updated_variants_by_data_type[data_type_key].update(updated_variants_by_id) + self._load_new_samples(metadata_path, **run_details) except Exception as e: logger.error(f'Error loading {run_details["run_version"]}: {e}') # Reset cached results for all projects, as seqr AFs will have changed for all projects when new data is added reset_cached_search_results(project=None) - backend_specific_call( - lambda *args: None, self._reload_dataset_type_shared_variant_annotations, lambda *args: None, - )(updated_families_by_data_type, updated_variants_by_data_type) - - @classmethod - def _reload_dataset_type_shared_variant_annotations(cls, updated_families_by_data_type, updated_variants_by_data_type): - for data_type_key, updated_families in updated_families_by_data_type.items(): - try: - cls._reload_shared_variant_annotations( - *data_type_key, updated_variants_by_data_type[data_type_key], exclude_families=updated_families, - ) - except Exception as e: - logger.error(f'Error reloading shared annotations for {"/".join(data_type_key)}: {e}') - @classmethod def _get_runs(cls, **kwargs): path = cls._run_path(lambda field: kwargs.get(field, '*') or '*') @@ -279,12 +256,9 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers logger.error(f'Error updating individuals sample qc {run_version}: {e}') # Reload saved variant JSON - update_function = backend_specific_call(None, None, cls._update_project_saved_variant_genotypes) - updated_variants_by_id = update_projects_saved_variant_json([ + update_projects_saved_variant_json([ (project.id, project.guid, project.name, project.genome_version, families) for project, families in families_by_project.items() - ], user_email=USER_EMAIL, dataset_type=dataset_type, update_function=update_function, samples=updated_samples, clickhouse_dataset_type=clickhouse_dataset_type) - - return search_data_type(dataset_type, sample_type), set(family_project_map.keys()), updated_variants_by_id + ], user_email=USER_EMAIL, dataset_type=dataset_type, update_function=cls._update_project_saved_variant_genotypes, samples=updated_samples, clickhouse_dataset_type=clickhouse_dataset_type) @classmethod def _is_internal_project(cls, project): @@ -440,96 +414,6 @@ def _update_project_saved_variant_genotypes(cls, project_id, genome_version, use updates.update({v.id: v for v in variants}) return updates - @classmethod - def _reload_shared_variant_annotations(cls, data_type, genome_version, updated_variants_by_id=None, exclude_families=None, chromosomes=None): - dataset_type = data_type.split('_')[0] - is_sv = dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS) - dataset_type = data_type.split('_')[0] if is_sv else data_type - db_genome_version = genome_version.replace('GRCh', '') - updated_annotation_samples = Sample.objects.filter( - is_active=True, dataset_type=dataset_type, - individual__family__project__genome_version=db_genome_version, - ) - if exclude_families: - updated_annotation_samples = updated_annotation_samples.exclude(individual__family__guid__in=exclude_families) - if is_sv: - updated_annotation_samples = updated_annotation_samples.filter(sample_type=data_type.split('_')[1]) - - variant_models = get_saved_variants( - genome_version, dataset_type=dataset_type, - family_guids=updated_annotation_samples.values_list('individual__family__guid', flat=True).distinct(), - ) - - variant_type_summary = f'{data_type} {genome_version} saved variants' - - if updated_variants_by_id: - fetched_variant_models = variant_models.filter(variant_id__in=updated_variants_by_id.keys()) - if fetched_variant_models: - logger.info(f'Reloading shared annotations for {len(fetched_variant_models)} fetched {variant_type_summary}') - for variant_model in fetched_variant_models: - updated_variant = { - k: v for k, v in updated_variants_by_id[variant_model.variant_id].items() - if k not in {'familyGuids', 'genotypes'} - } - variant_model.saved_variant_json.update(updated_variant) - SavedVariant.bulk_update_models(None, fetched_variant_models, ['saved_variant_json']) - - variant_models = variant_models.exclude(variant_id__in=updated_variants_by_id.keys()) - - chromosomes = cls._get_chroms_to_reload(chromosomes, dataset_type, genome_version, variant_models) - if chromosomes is None: - return - - for chrom in chromosomes: - cls._reload_shared_variant_annotations_by_chrom(chrom, variant_models, data_type, genome_version, variant_type_summary) - - @staticmethod - def _get_chroms_to_reload(chromosomes, dataset_type, genome_version, variant_models): - if dataset_type != Sample.DATASET_TYPE_VARIANT_CALLS: - return [None] - if chromosomes: - return chromosomes - - num_reload = variant_models.count() - if genome_version == GENOME_VERSION_LOOKUP[GENOME_VERSION_GRCh38] and num_reload > MAX_RELOAD_VARIANTS: - logger.info(f'Skipped reloading all {num_reload} saved variant annotations for {dataset_type} {genome_version}') - return None - - return CHROMOSOMES - - @classmethod - def _reload_shared_variant_annotations_by_chrom(cls, chrom, variant_models, data_type, genome_version, variant_type_summary): - if chrom: - variant_models = variant_models.filter(xpos__gte=get_xpos(chrom, MIN_POS), xpos__lte=get_xpos(chrom, MAX_POS)) - - chrom_summary = f' in chromosome {chrom}' if chrom else '' - if not variant_models: - logger.info(f'No additional {variant_type_summary} to update{chrom_summary}') - return - - variants_by_id = defaultdict(list) - for v in variant_models: - variants_by_id[v.variant_id].append(v) - - logger.info(f'Reloading shared annotations for {len(variant_models)} {variant_type_summary}{chrom_summary} ({len(variants_by_id)} unique)') - - variant_ids = sorted(variants_by_id.keys()) - if chrom: - variant_ids = sorted([parse_valid_variant_id(variant_id) for variant_id in variant_ids]) - - for i in range(0, len(variant_ids), MAX_LOOKUP_VARIANTS): - updated_variants = hail_variant_multi_lookup(USER_EMAIL, variant_ids[i:i+MAX_LOOKUP_VARIANTS], data_type, genome_version) - logger.info(f'Fetched {len(updated_variants)} additional variants{chrom_summary}') - - updated_variant_models = [] - for variant in updated_variants: - for variant_model in variants_by_id[variant['variantId']]: - variant_model.saved_variant_json.update(variant) - updated_variant_models.append(variant_model) - - SavedVariant.bulk_update_models(None, updated_variant_models, ['saved_variant_json']) - -reload_shared_variant_annotations = Command._reload_shared_variant_annotations update_individuals_sample_qc = Command._update_individuals_sample_qc get_pipeline_runs = Command._get_runs diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index c35eb6a1ed..60f4cc3dfc 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -451,12 +451,12 @@ def _parse_variant_items(search_json): def parse_variant_id(variant_id): try: - return parse_valid_variant_id(variant_id) + return _parse_valid_variant_id(variant_id) except (KeyError, ValueError): return None -def parse_valid_variant_id(variant_id): +def _parse_valid_variant_id(variant_id): chrom, pos, ref, alt = variant_id.split('-') chrom = format_chrom(chrom) pos = int(pos) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 5f71832f7a..9c24d50d8a 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -39,7 +39,6 @@ def update_projects_saved_variant_json(projects, user_email, update_function=Non success = {} skipped = {} error = {} - updated_variants_by_id = {} logger.info(f'Reloading saved variants in {len(projects)} projects') for project_id, project_guid, project_name, genome_version, family_guids in tqdm(projects, unit=' project'): try: @@ -51,7 +50,6 @@ def update_projects_saved_variant_json(projects, user_email, update_function=Non success[project_name] = len(updated_saved_variants) family_summary = f' in {len(family_guids)} families' if family_guids else '' logger.info(f'Updated {len(updated_saved_variants)} variants{family_summary} for project {project_name}') - updated_variants_by_id.update({v.variant_id: v.saved_variant_json for v in updated_saved_variants.values()}) except Exception as e: traceback_message = traceback.format_exc() logger.error(traceback_message) @@ -68,7 +66,6 @@ def update_projects_saved_variant_json(projects, user_email, update_function=Non logger.info(f'{len(error)} failed projects') for k, v in error.items(): logger.info(f' {k}: {v}') - return updated_variants_by_id def get_saved_variants(genome_version, project_id=None, family_guids=None, dataset_type=None, clickhouse_dataset_type=None): From 388f5be2f977817251a73f9044bb57a73d53afcd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 12:54:59 -0400 Subject: [PATCH 04/89] clean up backend specific calls --- seqr/management/commands/reload_saved_variant_json.py | 2 +- .../commands/transfer_families_to_different_project.py | 2 +- seqr/utils/search/add_data_utils.py | 1 - seqr/views/apis/data_manager_api.py | 1 + seqr/views/apis/family_api.py | 4 ++-- seqr/views/apis/saved_variant_api.py | 4 ++-- seqr/views/apis/summary_data_api.py | 1 - seqr/views/react_app.py | 2 +- seqr/views/utils/anvil_metadata_utils.py | 2 +- seqr/views/utils/individual_utils.py | 2 +- seqr/views/utils/orm_to_json_utils.py | 3 +-- seqr/views/utils/variant_utils.py | 10 +++++----- 12 files changed, 16 insertions(+), 18 deletions(-) diff --git a/seqr/management/commands/reload_saved_variant_json.py b/seqr/management/commands/reload_saved_variant_json.py index 5b38509d02..9e075a21b9 100644 --- a/seqr/management/commands/reload_saved_variant_json.py +++ b/seqr/management/commands/reload_saved_variant_json.py @@ -20,7 +20,7 @@ def add_arguments(self, parser): parser.add_argument('--family-guid', help='optional family to reload variants for') def handle(self, *args, **options): - backend_specific_call(lambda: True, lambda: True, _clickhouse_error)() + backend_specific_call(lambda: True, _clickhouse_error)() projects_to_process = options['projects'] family_guid = options['family_guid'] diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index 80bd0fdb2a..0861998b4f 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -62,7 +62,7 @@ def handle(self, *args, **options): ] logger.info(f'Skipping {num_found - len(families)} families with analysis groups in the project: {", ".join(group_families)}') - backend_specific_call(lambda *args: None, _disable_search, _disable_search)(families, from_project) + backend_specific_call(lambda *args: None, _disable_search)(families, from_project) for variant_tag_type in VariantTagType.objects.filter(project=from_project): variant_tags = VariantTag.objects.filter(saved_variants__family__in=families, variant_tag_type=variant_tag_type) diff --git a/seqr/utils/search/add_data_utils.py b/seqr/utils/search/add_data_utils.py index 3241d42159..51124edbec 100644 --- a/seqr/utils/search/add_data_utils.py +++ b/seqr/utils/search/add_data_utils.py @@ -31,7 +31,6 @@ def add_new_es_search_samples(request_json, project, user, notify=False, expecte sample_ids, sample_type, sample_data = backend_specific_call( validate_es_index_metadata_and_get_samples, _no_es_backend_error, - _no_es_backend_error, )(request_json, project) if not sample_ids: raise ValueError('No samples found. Make sure the specified caller type is correct') diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 5992cd9e2e..6f6d3e1d9b 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -453,6 +453,7 @@ def _raise_backend_not_implemented(*args, **kwargs): def _trigger_data_update(project, clickhouse_func=_raise_backend_not_implemented, **kwargs): kwargs = {_to_snake_case(k): v for k, v in kwargs.items()} + # TODO info = backend_specific_call(_raise_backend_not_implemented, _trigger_dag, clickhouse_func)(project, **kwargs) return create_json_response({'info': [info]}) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index af71b1b34c..4874adcd33 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -53,13 +53,13 @@ def family_page_data(request, family_guid): add_families_context(response, families, project.guid, request.user, is_analyst, has_case_review_perm) family_response = response['familiesByGuid'][family_guid] - additional_fields = backend_specific_call([],[],['key', 'dataset_type']) + additional_fields = backend_specific_call([],['key', 'dataset_type']) discovery_variants = family.savedvariant_set.filter(varianttag__variant_tag_type__category=DISCOVERY_CATEGORY).values( 'xpos', 'xpos_end', *additional_fields, svType=F('saved_variant_json__svType'), transcripts=F('saved_variant_json__transcripts'), ) gene_ids = backend_specific_call( - _variants_gene_ids, _variants_gene_ids, _clickhouse_variants_gene_ids, + _variants_gene_ids, _clickhouse_variants_gene_ids, )(discovery_variants, project.genome_version) discovery_variant_intervals = [dict(zip( ['chrom', 'start', 'end_chrom', 'end', 'svType', 'hasSvType'], diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 0044ea276b..e6ef867254 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -127,7 +127,7 @@ def _create_variant_note(saved_variants, note_json, user, genome_version): response = {} if note_json.get('saveAsGeneNote'): - gene_id = backend_specific_call(_variant_gene_id, _variant_gene_id, _clickhouse_variant_gene_id)(saved_variants[0], genome_version) + gene_id = backend_specific_call(_variant_gene_id, _clickhouse_variant_gene_id)(saved_variants[0], genome_version) create_model_from_json(GeneNote, {'note': note_json.get('note'), 'gene_id': gene_id}, user) response['genesById'] = {gene_id: { 'notes': get_json_for_gene_notes_by_gene_id([gene_id], user)[gene_id], @@ -320,7 +320,7 @@ def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=Vari @login_and_policies_required def update_saved_variant_json(request, project_guid): - backend_specific_call(lambda: True, _backend_error, _backend_error)() + backend_specific_call(lambda: True, _backend_error)() project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) reset_cached_search_results(project) try: diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 8e747c3a02..96a79a4776 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -115,7 +115,6 @@ def saved_variants_page(request, tag): if gene: gene_filter = Q(saved_variant_json__transcripts__has_key=gene) | backend_specific_call( - lambda *args: Q(), lambda *args: Q(), _saved_variant_with_clickhouse_gene_q, )(saved_variant_models, gene) diff --git a/seqr/views/react_app.py b/seqr/views/react_app.py index 84744cb5cd..10164a0679 100644 --- a/seqr/views/react_app.py +++ b/seqr/views/react_app.py @@ -60,7 +60,7 @@ def render_app_html(request, additional_json=None, include_user=True, status=200 'version': '{}-{}'.format(SEQR_VERSION, ui_version), 'hijakEnabled': DEBUG or False, 'oauthLoginProvider': SOCIAL_AUTH_PROVIDER, - 'elasticsearchEnabled': backend_specific_call(True, False, False), + 'elasticsearchEnabled': backend_specific_call(True, False), 'vlmEnabled': bool(VLM_CLIENT_ID), 'warningMessages': [message.json() for message in WarningMessage.objects.all()], 'anvilLoadingDelayDate': ANVIL_LOADING_DELAY_EMAIL_START_DATE if should_show_loading_delay else None, diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 4bc6826e4d..ec867c784d 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -347,7 +347,7 @@ def _get_parsed_saved_discovery_variants_by_family( variants = [] gene_ids = set() variant_json_by_guid = backend_specific_call( - _get_variant_json_by_guid, _get_variant_json_by_guid, _get_clickhouse_variant_json_by_guid, + _get_variant_json_by_guid, _get_clickhouse_variant_json_by_guid, )(project_saved_variants, include_clinvar) for variant in project_saved_variants: chrom, pos = get_chrom_pos(variant.xpos) diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index 733de7657e..77b1a69ed3 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -203,7 +203,7 @@ def check_project_individuals_deletable(project, individual_guids=None): if individual_guids is not None: individuals_to_delete = individuals_to_delete.filter(guid__in=individual_guids) - errors = backend_specific_call(_validate_no_submissions, _validate_no_sumissions_no_search_samples, _validate_no_sumissions_no_search_samples)(individuals_to_delete) + errors = backend_specific_call(_validate_no_submissions, _validate_no_sumissions_no_search_samples)(individuals_to_delete) return errors, individuals_to_delete diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 1f6ecb75dc..579f60580b 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -424,7 +424,6 @@ def get_json_for_saved_variants(saved_variants, add_details=False, additional_mo if add_details: from seqr.utils.search.utils import backend_specific_call additional_fields += ['saved_variant_json'] + backend_specific_call( - lambda x: [], lambda x: [], lambda gv: ['key', 'genotypes', 'dataset_type'] + ([] if gv else ['family__project__genome_version']), )(genome_version) @@ -436,7 +435,7 @@ def get_json_for_saved_variants(saved_variants, add_details=False, additional_mo if add_details: from seqr.utils.search.utils import backend_specific_call - backend_specific_call(lambda *args: None, lambda *args: None, _add_clickhouse_annotations)(results, genome_version) + backend_specific_call(lambda *args: None, _add_clickhouse_annotations)(results, genome_version) for result in results: result.update({k: v for k, v in result.pop('savedVariantJson').items() if k not in result}) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 9c24d50d8a..a4a4b59ba5 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -174,7 +174,7 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use new_variant_keys = set(family_variant_data.keys()) - set(saved_variant_map.keys()) if new_variant_keys: new_variant_data = _search_new_saved_variants(new_variant_keys, user, genome_version) if load_new_variant_data else backend_specific_call( - lambda o, _: o, lambda o, _: o, _get_clickhouse_variant_keys, + lambda o, _: o, _get_clickhouse_variant_keys, )({k: v for k, v in family_variant_data.items() if k in new_variant_keys}, genome_version) new_variant_models = [] for (family_id, variant_id), variant in new_variant_data.items(): @@ -193,7 +193,7 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use } variant_genes_by_id = backend_specific_call( - _get_saved_variant_genes, _get_saved_variant_genes, _get_clickhouse_saved_variant_genes, + _get_saved_variant_genes, _get_clickhouse_saved_variant_genes, )(saved_variant_map.values(), genome_version) update_tags = [] @@ -280,7 +280,7 @@ def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], user: U search_variants_by_id = { v['variantId']: v for v in backend_specific_call( - get_variants_for_variant_ids, get_variants_for_variant_ids, _get_clickhouse_variants, + get_variants_for_variant_ids, _get_clickhouse_variants, )(families=list(families_by_id.values()), variant_ids=list(variant_families.keys()), user=user, genome_version=genome_version) } @@ -377,7 +377,7 @@ def _saved_variant_genes_transcripts(variants): for var in variant: for gene_id, transcripts in var.get('transcripts', {}).items(): gene_ids.add(gene_id) - if backend_specific_call(lambda v: True, _requires_transcript_metadata, _requires_transcript_metadata)(variant): + if backend_specific_call(lambda v: True, _requires_transcript_metadata)(variant): transcript_ids.update([t['transcriptId'] for t in transcripts if t.get('transcriptId')]) for family_guid in var['familyGuids']: family_genes[family_guid].update(var.get('transcripts', {}).keys()) @@ -553,7 +553,7 @@ def get_variants_response(request, saved_variants, response_variants=None, add_a response['familiesByGuid'][family_guid] = {} response['familiesByGuid'][family_guid].update(data) - backend_specific_call(lambda response: response, _add_sample_count_stats, _add_sample_count_stats)(response) + backend_specific_call(lambda response: response, _add_sample_count_stats)(response) return response From 09a81b5f8fd0560dbaeff93bff4483d85f9f61ea Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 13:01:26 -0400 Subject: [PATCH 05/89] remove unused dag triggers --- seqr/urls.py | 3 +-- seqr/views/apis/data_manager_api.py | 27 +++---------------- .../TriggerSearchDataUpdatePages.jsx | 10 ------- 3 files changed, 5 insertions(+), 35 deletions(-) diff --git a/seqr/urls.py b/seqr/urls.py index f93c3a603a..1ce292f0d2 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -125,7 +125,7 @@ from seqr.views.apis.data_manager_api import elasticsearch_status, delete_index, \ update_rna_seq, load_rna_seq_sample_data, proxy_to_kibana, load_phenotype_prioritization_data, \ validate_callset, get_loaded_projects, load_data, loading_vcfs, proxy_to_luigi, \ - trigger_delete_project, trigger_delete_family, trigger_update_search_reference_data + trigger_delete_project, trigger_delete_family from seqr.views.apis.report_api import \ anvil_export, \ family_metadata, \ @@ -342,7 +342,6 @@ 'data_management/add_igv': receive_bulk_igv_table_handler, 'data_management/trigger_delete_project': trigger_delete_project, 'data_management/trigger_delete_family': trigger_delete_family, - 'data_management/trigger_update_search_reference_data': trigger_update_search_reference_data, 'summary_data/saved_variants/(?P[^/]+)': saved_variants_page, 'summary_data/hpo/(?P[^/]+)': hpo_summary_data, diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 6f6d3e1d9b..6058286a75 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -22,7 +22,7 @@ from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.vcf_utils import validate_vcf_and_get_samples, get_vcf_list -from seqr.views.utils.airflow_utils import trigger_airflow_data_loading, trigger_airflow_dag, is_airflow_enabled +from seqr.views.utils.airflow_utils import trigger_airflow_data_loading, trigger_airflow_dag, is_airflow_enabled # TODO still used anywhere? from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS from seqr.views.utils.dataset_utils import load_rna_seq, load_phenotype_prioritization_data_file, RNA_DATA_TYPE_CONFIGS, \ post_process_rna_data, convert_django_meta_to_http_headers @@ -438,38 +438,19 @@ def trigger_delete_family(request): request_json = json.loads(request.body) family_guid = request_json.pop('family') project = Project.objects.get(family__guid=family_guid) - return _trigger_data_update(project, family_guid=family_guid, dag_id='DELETE_FAMILIES', **request_json) - - -@data_manager_required -def trigger_update_search_reference_data(request): - request_json = json.loads(request.body) - return _trigger_data_update(project=None, dag_id='UPDATE_REFERENCE_DATASETS', **request_json) + return _trigger_data_update(project, clickhouse_func=_raise_backend_not_implemented, family_guid=family_guid, dag_id='DELETE_FAMILIES', **request_json) def _raise_backend_not_implemented(*args, **kwargs): raise ErrorsWarningsException(['This functionality is not available in the current search backend']) -def _trigger_data_update(project, clickhouse_func=_raise_backend_not_implemented, **kwargs): +def _trigger_data_update(project, clickhouse_func, **kwargs): kwargs = {_to_snake_case(k): v for k, v in kwargs.items()} - # TODO - info = backend_specific_call(_raise_backend_not_implemented, _trigger_dag, clickhouse_func)(project, **kwargs) + info = backend_specific_call(_raise_backend_not_implemented, clickhouse_func)(project, **kwargs) return create_json_response({'info': [info]}) -def _trigger_dag(project, dag_id=None, family_guid=None, **kwargs): - if not is_airflow_enabled(): - raise PermissionDenied() - if family_guid: - kwargs['family_guids'] = [family_guid] - try: - dag_variables = trigger_airflow_dag(dag_id, project, **kwargs) - except Exception as e: - raise ErrorsWarningsException([str(e)]) - return f'Triggered DAG {dag_id} with variables: {json.dumps(dag_variables)}' - - # Hop-by-hop HTTP response headers shouldn't be forwarded. # More info at: http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.1 EXCLUDE_HTTP_RESPONSE_HEADERS = { diff --git a/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx b/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx index 7aa4ca250b..6a0e6fdfaa 100644 --- a/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx +++ b/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx @@ -9,7 +9,6 @@ import { DATASET_TYPE_SNV_INDEL_CALLS, DATASET_TYPE_SV_CALLS, DATASET_TYPE_MITO_CALLS, - GENOME_VERSION_FIELD, } from 'shared/utils/constants' import { snakecaseToTitlecase } from 'shared/utils/stringUtils' @@ -46,10 +45,6 @@ const FAMILY_FIELDS = [ }, DATASET_TYPE_FIELD, ] -const REFERENCE_DATASET_FIELDS = [ - { ...GENOME_VERSION_FIELD, validate: validators.required }, - DATASET_TYPE_FIELD, -] const TriggerSearchDataUpdateForm = ({ path, fields }) => ( ( ) -const TriggerUpdateReferenceDataset = () => ( - -) - export default [ { path: 'delete_search_project', component: TriggerDeleteProjects }, { path: 'delete_search_family', component: TriggerDeleteFamilies }, - { path: 'update_search_reference_data', component: TriggerUpdateReferenceDataset }, ] From afbf6694b483cd7827cc12e43e90cd0bd2f023c9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 13:03:41 -0400 Subject: [PATCH 06/89] todos --- seqr/views/apis/data_manager_api.py | 2 +- seqr/views/utils/airflow_utils.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 6058286a75..510ef8c952 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -22,7 +22,7 @@ from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.vcf_utils import validate_vcf_and_get_samples, get_vcf_list -from seqr.views.utils.airflow_utils import trigger_airflow_data_loading, trigger_airflow_dag, is_airflow_enabled # TODO still used anywhere? +from seqr.views.utils.airflow_utils import trigger_airflow_data_loading from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS from seqr.views.utils.dataset_utils import load_rna_seq, load_phenotype_prioritization_data_file, RNA_DATA_TYPE_CONFIGS, \ post_process_rna_data, convert_django_meta_to_http_headers diff --git a/seqr/views/utils/airflow_utils.py b/seqr/views/utils/airflow_utils.py index 634903a192..09d18017b2 100644 --- a/seqr/views/utils/airflow_utils.py +++ b/seqr/views/utils/airflow_utils.py @@ -24,6 +24,7 @@ class DagRunningException(Exception): def is_airflow_enabled(): + # TODO unused return bool(AIRFLOW_WEBSERVER_URL) @@ -53,6 +54,7 @@ def trigger_airflow_data_loading(*args, user: User, success_message: str, succes def trigger_airflow_dag(dag_id: str, project: Project, dataset_type: str, genome_version: str = None, **kwargs): + # TODO unused variables = format_loading_pipeline_variables( [project] if project else [], project.genome_version if project else genome_version, From 0baa7197190c9d7d392b5c3ac169ca7837724474 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 13:59:16 -0400 Subject: [PATCH 07/89] remve airflow from deactivate project --- seqr/management/commands/deactivate_project_search.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/seqr/management/commands/deactivate_project_search.py b/seqr/management/commands/deactivate_project_search.py index 57088c089e..19001db8ff 100644 --- a/seqr/management/commands/deactivate_project_search.py +++ b/seqr/management/commands/deactivate_project_search.py @@ -3,7 +3,6 @@ from clickhouse_search.search import delete_clickhouse_project from seqr.models import Project, Sample from seqr.utils.search.utils import backend_specific_call -from seqr.views.utils.airflow_utils import is_airflow_enabled, trigger_airflow_dag, DELETE_PROJECTS_DAG_NAME import logging logger = logging.getLogger(__name__) @@ -27,15 +26,9 @@ def handle(self, *args, **options): dataset_types = Sample.objects.filter(guid__in=updated).values_list('dataset_type', 'sample_type').order_by('dataset_type').distinct() for dataset_type, sample_type in dataset_types: backend_specific_call( - lambda *args, **kwargs: True, self._trigger_delete_dag, self._delete_clickhouse_project, + lambda *args, **kwargs: True, self._delete_clickhouse_project, )(project, dataset_type, sample_type) - @staticmethod - def _trigger_delete_dag(project, dataset_type, sample_type): - if is_airflow_enabled(): - trigger_airflow_dag(DELETE_PROJECTS_DAG_NAME, project, 'SNV_INDEL') - logger.info(f'Successfully triggered {DELETE_PROJECTS_DAG_NAME} DAG for {dataset_type} {project.guid}') - @staticmethod def _delete_clickhouse_project(project, dataset_type, sample_type): if dataset_type == Sample.DATASET_TYPE_SV_CALLS and sample_type == Sample.SAMPLE_TYPE_WES: From 723254db5c4d7d90eb6af8a75878990adc3f0bd1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 14:32:04 -0400 Subject: [PATCH 08/89] remove dag trigger from transfer families --- .../transfer_families_to_different_project.py | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index 0de4896509..dc46d104ee 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -1,16 +1,13 @@ -from collections import defaultdict - from django.core.management.base import BaseCommand from seqr.models import Project, Family, VariantTag, VariantTagType, Sample from seqr.utils.search.utils import backend_specific_call -from seqr.views.utils.airflow_utils import trigger_airflow_dag, DELETE_FAMILIES_DAG_NAME, DagRunningException import logging logger = logging.getLogger(__name__) -def _disable_search(families, from_project): +def _disable_search(families): search_samples = Sample.objects.filter(is_active=True, individual__family__in=families) updated_family_dataset_types = None if search_samples: @@ -23,19 +20,6 @@ def _disable_search(families, from_project): ) return updated_family_dataset_types -def _disable_search_trigger_delete_families_dags(families, from_project): - updated_family_dataset_types = _disable_search(families, from_project) - updated_families_by_dataset_type = defaultdict(list) - for dataset_type, family_guid in updated_family_dataset_types: - updated_families_by_dataset_type[dataset_type].append(family_guid) - - for dataset_type, family_guids in sorted(updated_families_by_dataset_type.items()): - try: - trigger_airflow_dag(DELETE_FAMILIES_DAG_NAME, from_project, dataset_type, family_guids=sorted(family_guids)) - logger.info(f'Successfully triggered DELETE_FAMILIES DAG for {len(family_guids)} {dataset_type} families') - except Exception as e: - logger_call = logger.warning if isinstance(e, DagRunningException) else logger.error - logger_call(str(e)) class Command(BaseCommand): def add_arguments(self, parser): @@ -64,7 +48,7 @@ def handle(self, *args, **options): ] logger.info(f'Skipping {num_found - len(families)} families with analysis groups in the project: {", ".join(group_families)}') - backend_specific_call(lambda *args: None, _disable_search)(families, from_project) + backend_specific_call(lambda *args: None, _disable_search)(families) for variant_tag_type in VariantTagType.objects.filter(project=from_project): variant_tags = VariantTag.objects.filter(saved_variants__family__in=families, variant_tag_type=variant_tag_type) From 61fbf4a08cf5d63c86f2f599fc9e4358357763ae Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 14:34:14 -0400 Subject: [PATCH 09/89] remove generic airflow dag triggering --- seqr/utils/search/add_data_utils.py | 4 ++-- seqr/views/utils/airflow_utils.py | 31 +---------------------------- 2 files changed, 3 insertions(+), 32 deletions(-) diff --git a/seqr/utils/search/add_data_utils.py b/seqr/utils/search/add_data_utils.py index 51124edbec..f6e30f84d8 100644 --- a/seqr/utils/search/add_data_utils.py +++ b/seqr/utils/search/add_data_utils.py @@ -102,7 +102,7 @@ def update_airtable_loading_tracking_status(project, status, additional_update=N update={'Status': status, **(additional_update or {})}, ) -def format_loading_pipeline_variables( +def _format_loading_pipeline_variables( projects: list[Project], genome_version: str, dataset_type: str, sample_type: str = None, **kwargs ): variables = { @@ -118,7 +118,7 @@ def format_loading_pipeline_variables( def prepare_data_loading_request(projects: list[Project], individual_ids: list[int], sample_type: str, dataset_type: str, genome_version: str, data_path: str, user: User, pedigree_dir: str, raise_pedigree_error: bool = False, skip_validation: bool = False, skip_check_sex_and_relatedness: bool = False, vcf_sample_id_map=None): - variables = format_loading_pipeline_variables( + variables = _format_loading_pipeline_variables( projects, genome_version, dataset_type, diff --git a/seqr/views/utils/airflow_utils.py b/seqr/views/utils/airflow_utils.py index 09d18017b2..2bc1d089bf 100644 --- a/seqr/views/utils/airflow_utils.py +++ b/seqr/views/utils/airflow_utils.py @@ -4,17 +4,14 @@ from google.auth.transport.requests import AuthorizedSession import json -from seqr.models import Project from seqr.utils.communication_utils import safe_post_to_slack -from seqr.utils.search.add_data_utils import prepare_data_loading_request, format_loading_pipeline_variables +from seqr.utils.search.add_data_utils import prepare_data_loading_request from seqr.utils.logging_utils import SeqrLogger from settings import AIRFLOW_WEBSERVER_URL, SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL logger = SeqrLogger(__name__) LOADING_PIPELINE_DAG_NAME = 'LOADING_PIPELINE' -DELETE_FAMILIES_DAG_NAME = 'DELETE_FAMILIES' -DELETE_PROJECTS_DAG_NAME = 'DELETE_PROJECTS' AIRFLOW_AUTH_SCOPE = "https://www.googleapis.com/auth/cloud-platform" SEQR_V3_PEDIGREE_GS_PATH = 'gs://seqr-loading-temp/v3.1' @@ -23,11 +20,6 @@ class DagRunningException(Exception): pass -def is_airflow_enabled(): - # TODO unused - return bool(AIRFLOW_WEBSERVER_URL) - - def trigger_airflow_data_loading(*args, user: User, success_message: str, success_slack_channel: str, error_message: str, is_internal: bool = False, **kwargs): success = True @@ -53,21 +45,6 @@ def trigger_airflow_data_loading(*args, user: User, success_message: str, succes return success -def trigger_airflow_dag(dag_id: str, project: Project, dataset_type: str, genome_version: str = None, **kwargs): - # TODO unused - variables = format_loading_pipeline_variables( - [project] if project else [], - project.genome_version if project else genome_version, - dataset_type, - **kwargs - ) - _check_dag_running_state(dag_id) - _update_variables(variables, dag_id) - _wait_for_dag_variable_update(variables, dag_id) - _trigger_dag(dag_id) - return variables - - def _send_load_data_slack_msg(messages: list[str], channel: str, dag: dict): message = '\n\n '.join(messages) message_content = f"""{message} @@ -102,12 +79,6 @@ def _wait_for_dag_variable_update_via_tasks(projects: list[str], dag_name: str): dag_projects = _get_task_ids(dag_name) -def _wait_for_dag_variable_update(variables: dict, dag_name: str): - existing_variables = _get_variables(dag_name) - while any(existing_variables.get(k) != v for k, v in variables.items()): - existing_variables = _get_variables(dag_name) - - def _update_variables(val: dict, dag_name: str): endpoint = f'variables/{dag_name}' val_str = json.dumps(val) From 05e31b46ced9284f1ddae1abad3272a6b10f4256 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 14:43:15 -0400 Subject: [PATCH 10/89] placeholder delete family function --- clickhouse_search/search.py | 4 ++++ .../commands/transfer_families_to_different_project.py | 9 +++++---- seqr/views/apis/data_manager_api.py | 6 +++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 1618dc141b..4b75182f2e 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -629,3 +629,7 @@ def delete_clickhouse_project(project, dataset_type=None, **kwargs): cursor.execute(f'SYSTEM WAIT VIEW "{view_name}"') cursor.execute(f'SYSTEM RELOAD DICTIONARY "{table_base}/gt_stats_dict"') return f'Deleted all {dataset_type} search data for project {project.name}' + + +def delete_clickhouse_family(project, family_guid, dataset_type=None, **kwargs): + return f'Clickhouse does not support deleting individual families from project. Manually delete {dataset_type} data for {family_guid} in project {project.guid}' diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index dc46d104ee..09ea643181 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -1,5 +1,6 @@ from django.core.management.base import BaseCommand +from clickhouse_search.search import delete_clickhouse_family from seqr.models import Project, Family, VariantTag, VariantTagType, Sample from seqr.utils.search.utils import backend_specific_call @@ -7,9 +8,8 @@ logger = logging.getLogger(__name__) -def _disable_search(families): +def _disable_search(families, from_project): search_samples = Sample.objects.filter(is_active=True, individual__family__in=families) - updated_family_dataset_types = None if search_samples: updated_families = search_samples.values_list("individual__family__family_id", flat=True).distinct() updated_family_dataset_types = list(search_samples.values_list('dataset_type', 'individual__family__guid').distinct()) @@ -18,7 +18,8 @@ def _disable_search(families): logger.info( f'Disabled search for {num_updated} samples in the following {len(updated_families)} families: {family_summary}' ) - return updated_family_dataset_types + for dataset_type, family_guid in updated_family_dataset_types: + logger.info(delete_clickhouse_family(from_project, family_guid=family_guid, dataset_type=dataset_type)) class Command(BaseCommand): @@ -48,7 +49,7 @@ def handle(self, *args, **options): ] logger.info(f'Skipping {num_found - len(families)} families with analysis groups in the project: {", ".join(group_families)}') - backend_specific_call(lambda *args: None, _disable_search)(families) + backend_specific_call(lambda *args: None, _disable_search)(families, from_project) for variant_tag_type in VariantTagType.objects.filter(project=from_project): variant_tags = VariantTag.objects.filter(saved_variants__family__in=families, variant_tag_type=variant_tag_type) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 510ef8c952..85ea92447f 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -13,7 +13,7 @@ from django.views.decorators.csrf import csrf_exempt from requests.exceptions import ConnectionError as RequestConnectionError -from clickhouse_search.search import delete_clickhouse_project +from clickhouse_search.search import delete_clickhouse_project, delete_clickhouse_family from seqr.utils.communication_utils import send_project_notification from seqr.utils.search.add_data_utils import prepare_data_loading_request, get_loading_samples_validator from seqr.utils.search.utils import get_search_backend_status, delete_search_backend_data, backend_specific_call @@ -429,7 +429,7 @@ def trigger_delete_project(request): project_guid = request_json.pop('project') project = Project.objects.get(guid=project_guid) return _trigger_data_update( - project, dag_id='DELETE_PROJECTS', clickhouse_func=delete_clickhouse_project, **request_json, + project, delete_clickhouse_project, **request_json, ) @@ -438,7 +438,7 @@ def trigger_delete_family(request): request_json = json.loads(request.body) family_guid = request_json.pop('family') project = Project.objects.get(family__guid=family_guid) - return _trigger_data_update(project, clickhouse_func=_raise_backend_not_implemented, family_guid=family_guid, dag_id='DELETE_FAMILIES', **request_json) + return _trigger_data_update(project, delete_clickhouse_family, family_guid=family_guid, **request_json) def _raise_backend_not_implemented(*args, **kwargs): From dc5ca1db80bf86932f85471c6919afcb10dc753f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 22 Aug 2025 09:58:31 -0400 Subject: [PATCH 11/89] remove hail search utils; --- seqr/utils/search/hail_search_utils.py | 175 --------- seqr/utils/search/hail_search_utils_tests.py | 381 ------------------- 2 files changed, 556 deletions(-) delete mode 100644 seqr/utils/search/hail_search_utils.py delete mode 100644 seqr/utils/search/hail_search_utils_tests.py diff --git a/seqr/utils/search/hail_search_utils.py b/seqr/utils/search/hail_search_utils.py deleted file mode 100644 index 0cc5f53226..0000000000 --- a/seqr/utils/search/hail_search_utils.py +++ /dev/null @@ -1,175 +0,0 @@ -from collections import defaultdict - -from django.db.models import F, Min, Case, When -from time import sleep -from urllib3.connectionpool import connection_from_url - -import requests -from reference_data.models import Omim, GeneConstraint, GENOME_VERSION_LOOKUP -from seqr.models import Sample, PhenotypePrioritization, Individual -from seqr.utils.search.constants import PRIORITIZED_GENE_SORT, X_LINKED_RECESSIVE -from settings import HAIL_BACKEND_SERVICE_HOSTNAME, HAIL_BACKEND_SERVICE_PORT - - -def _hail_backend_url(path): - return f'http://{HAIL_BACKEND_SERVICE_HOSTNAME}:{HAIL_BACKEND_SERVICE_PORT}/{path}' - -def _execute_search(search_body, user, path='search', exception_map=None, user_email=None, max_retries=0): - for _ in range(max_retries + 1): - response = requests.post(_hail_backend_url(path), json=search_body, headers={'From': user_email or user.email}, timeout=300) - if response.status_code == 200: - return response.json() - sleep(1) - - error = (exception_map or {}).get(response.status_code) or response.text or response.reason - raise requests.HTTPError(error, response=response) - - -def ping_hail_backend(): - response = connection_from_url(_hail_backend_url('status')).urlopen('HEAD', '/status', timeout=5, retries=3) - if response.status >= 400: - raise requests.HTTPError(f'{response.status}: {response.reason or response.text}', response=response) - - -def get_hail_variants(samples, search, user, previous_search_results, genome_version, sort=None, page=1, num_results=100, - gene_agg=False, **kwargs): - end_offset = num_results * page - search_body = _format_search_body(samples, genome_version, end_offset, search) - - frequencies = search_body.pop('freqs', None) - if frequencies and frequencies.get('callset'): - frequencies['seqr'] = frequencies.pop('callset') - - search_body.update({ - 'sort': sort, - 'sort_metadata': _get_sort_metadata(sort, samples), - 'frequencies': frequencies, - 'quality_filter': search_body.pop('qualityFilter', None), - **search_body.pop('parsed_locus') - }) - search_body.pop('skipped_samples', None) - - sv_data_types = [data_type for data_type in search_body['sample_data'] if data_type.startswith(Sample.DATASET_TYPE_SV_CALLS)] - if len(sv_data_types) > 1 and not gene_agg: - response_json = _execute_multi_sample_types_searches(sv_data_types, search_body, user) - else: - path = 'gene_counts' if gene_agg else 'search' - response_json = _execute_search(search_body, user, path) - - if gene_agg: - previous_search_results['gene_aggs'] = response_json - return response_json - - previous_search_results['total_results'] = response_json['total'] - previous_search_results['all_results'] = response_json['results'] - return response_json['results'][end_offset - num_results:end_offset] - - -def _execute_multi_sample_types_searches(sv_data_types, search_body, user): - sample_data_by_sample_data_type = defaultdict(lambda: defaultdict(list)) - for data_type, samples in search_body['sample_data'].items(): - if data_type in sv_data_types: - sample_type = samples[0]['sample_type'] - sample_data_by_sample_data_type[sample_type][data_type] = samples - else: - for sample in samples: - sample_data_by_sample_data_type[sample['sample_type']][data_type].append(sample) - - total = 0 - results = [] - for sample_type in sorted(sample_data_by_sample_data_type.keys()): - search_body['sample_data'] = sample_data_by_sample_data_type[sample_type] - sample_response_json = _execute_search(search_body, user) - total += sample_response_json['total'] - results += sample_response_json['results'] - - return {'total': total, 'results': sorted(results, key=lambda x: x[0]['_sort'] if isinstance(x, list) else x['_sort'])[:search_body['num_results']]} - - -def get_hail_variants_for_variant_ids(samples, genome_version, parsed_variant_ids, user, user_email=None): - search = { - 'variant_ids': [parsed_id for parsed_id in parsed_variant_ids.values() if parsed_id], - 'variant_keys': [variant_id for variant_id, parsed_id in parsed_variant_ids.items() if not parsed_id], - } - search_body = _format_search_body(samples, genome_version, len(parsed_variant_ids), search) - response_json = _execute_search(search_body, user, user_email=user_email) - - return response_json['results'] - - -def hail_variant_lookup(user, variant_id, data_type, genome_version=None, samples=None, **kwargs): - body = { - 'variant_id': variant_id, - 'data_type': data_type, - 'genome_version': GENOME_VERSION_LOOKUP[genome_version], - **kwargs, - } - if samples: - body['sample_data'] = _get_sample_data(samples).pop(data_type) - return _execute_search(body, user, path='lookup', exception_map={404: 'Variant not present in seqr'}) - - -def hail_variant_multi_lookup(user_email, variant_ids, data_type, genome_version): - body = {'genome_version': genome_version, 'data_type': data_type, 'variant_ids': variant_ids} - response_json = _execute_search(body, user=None, user_email=user_email, path='multi_lookup', max_retries=2) - return response_json['results'] - - -def _format_search_body(samples, genome_version, num_results, search): - search_body = { - 'genome_version': GENOME_VERSION_LOOKUP[genome_version], - 'num_results': num_results, - } - search_body.update(search) - search_body['sample_data'] = _get_sample_data(samples, **search_body) - search_body.pop('dataset_type', None) - search_body.pop('secondary_dataset_type', None) - return search_body - - -def search_data_type(dataset_type, sample_type): - return f'{dataset_type}_{sample_type}' if dataset_type == Sample.DATASET_TYPE_SV_CALLS else dataset_type - - -def _get_sample_data(samples, inheritance_filter=None, inheritance_mode=None, **kwargs): - sample_values = dict( - individual_guid=F('individual__guid'), - family_guid=F('individual__family__guid'), - project_guid=F('individual__family__project__guid'), - affected=F('individual__affected'), - ) - if inheritance_mode == X_LINKED_RECESSIVE: - sample_values['is_male'] = Case(When(individual__sex__in=Individual.MALE_SEXES, then=True), default=False) - sample_data = samples.order_by('guid').values('individual__individual_id', 'dataset_type', 'sample_type', **sample_values) - - custom_affected = (inheritance_filter or {}).pop('affected', None) - if custom_affected: - for s in sample_data: - s['affected'] = custom_affected.get(s['individual_guid']) or s['affected'] - - sample_data_by_data_type = defaultdict(list) - for s in sample_data: - dataset_type = s.pop('dataset_type') - s['sample_id'] = s.pop('individual__individual_id') # Note: set sample_id to individual_id - data_type_key = search_data_type(dataset_type, s['sample_type']) - sample_data_by_data_type[data_type_key].append(s) - - return sample_data_by_data_type - - -def _get_sort_metadata(sort, samples): - sort_metadata = None - if sort == 'in_omim': - sort_metadata = list(Omim.objects.filter(phenotype_mim_number__isnull=False, gene__isnull=False).values_list('gene__gene_id', flat=True)) - elif sort == 'constraint': - sort_metadata = { - agg['gene__gene_id']: agg['mis_z_rank'] + agg['pLI_rank'] for agg in - GeneConstraint.objects.values('gene__gene_id', 'mis_z_rank', 'pLI_rank') - } - elif sort == PRIORITIZED_GENE_SORT: - sort_metadata = { - agg['gene_id']: agg['min_rank'] for agg in PhenotypePrioritization.objects.filter( - individual__family_id=samples[0].individual.family_id, rank__lte=100, - ).values('gene_id').annotate(min_rank=Min('rank')) - } - return sort_metadata diff --git a/seqr/utils/search/hail_search_utils_tests.py b/seqr/utils/search/hail_search_utils_tests.py deleted file mode 100644 index b1e1700687..0000000000 --- a/seqr/utils/search/hail_search_utils_tests.py +++ /dev/null @@ -1,381 +0,0 @@ -from copy import deepcopy -from django.test import TestCase -import json -import mock -from requests import HTTPError -import responses - -from seqr.models import Family, Project -from seqr.utils.search.utils import get_variant_query_gene_counts, query_variants, get_single_variant, \ - get_variants_for_variant_ids, variant_lookup, sv_variant_lookup, InvalidSearchException -from seqr.utils.search.search_utils_tests import SearchTestHelper -from hail_search.test_utils import get_hail_search_body, EXPECTED_SAMPLE_DATA, FAMILY_1_SAMPLE_DATA, \ - ALL_AFFECTED_SAMPLE_DATA, CUSTOM_AFFECTED_SAMPLE_DATA, HAIL_BACKEND_VARIANTS, \ - LOCATION_SEARCH, EXCLUDE_LOCATION_SEARCH, VARIANT_ID_SEARCH, RSID_SEARCH, GENE_COUNTS, FAMILY_2_VARIANT_SAMPLE_DATA, \ - FAMILY_2_MITO_SAMPLE_DATA, EXPECTED_SAMPLE_DATA_WITH_SEX, VARIANT_LOOKUP_VARIANT, FAMILY_11_SAMPLE_WGS, \ - GCNV_VARIANT4, SV_VARIANT2 - -MOCK_HOST = 'test-hail-host' -MOCK_ORIGIN = f'http://{MOCK_HOST}' - -SV_WGS_SAMPLE_DATA = [{ - 'individual_guid': 'I000018_na21234', 'family_guid': 'F000014_14', 'project_guid': 'R0004_non_analyst_project', - 'affected': 'A', 'sample_id': 'NA21234', 'sample_type': 'WGS', -}, { - 'individual_guid': 'I000019_na21987', 'family_guid': 'F000014_14', 'project_guid': 'R0004_non_analyst_project', - 'affected': 'A', 'sample_id': 'NA21987', 'sample_type': 'WGS', -}, { - 'individual_guid': 'I000021_na21654', 'family_guid': 'F000014_14', 'project_guid': 'R0004_non_analyst_project', - 'affected': 'N', 'sample_id': 'NA21654', 'sample_type': 'WGS', -}] -SV_WES_SAMPLE_DATA = FAMILY_2_VARIANT_SAMPLE_DATA['SNV_INDEL'] - -EXPECTED_MITO_SAMPLE_DATA = deepcopy(FAMILY_2_MITO_SAMPLE_DATA) -EXPECTED_MITO_SAMPLE_DATA['MITO'][0].update({'individual_guid': 'I000004_hg00731', 'sample_id': 'HG00731', 'affected': 'A'}) - -ALL_EXPECTED_SAMPLE_DATA = {**EXPECTED_SAMPLE_DATA, **EXPECTED_MITO_SAMPLE_DATA} - - -@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') -@mock.patch('seqr.utils.search.hail_search_utils.HAIL_BACKEND_SERVICE_HOSTNAME', MOCK_HOST) -class HailSearchUtilsTests(SearchTestHelper, TestCase): - databases = ['default', 'reference_data'] - fixtures = ['users', '1kg_project', 'reference_data'] - - def setUp(self): - Project.objects.update(genome_version='37') - super(HailSearchUtilsTests, self).set_up() - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/search', status=200, json={ - 'results': HAIL_BACKEND_VARIANTS, 'total': 5, - }) - - def _test_minimal_search_call(self, expected_search_body=None, call_offset=-1, url_path='search', sample_data=ALL_EXPECTED_SAMPLE_DATA, **kwargs): - expected_search = expected_search_body or get_hail_search_body(genome_version='GRCh37', sample_data=sample_data, **kwargs) - - executed_request = responses.calls[call_offset].request - self.assertEqual(executed_request.headers.get('From'), 'test_user@broadinstitute.org') - self.assertEqual(executed_request.url.split('/')[-1], url_path) - self.assertDictEqual(json.loads(executed_request.body), expected_search) - - def _test_expected_search_call(self, search_fields=None, gene_ids=None, intervals=None, exclude_intervals=False, - rs_ids=None, variant_ids=None, dataset_type=None, secondary_dataset_type=None, - frequencies=None, inheritance_mode='de_novo', inheritance_filter=None, - quality_filter=None, sort='xpos', sort_metadata=None, annotations=None, - annotations_secondary=None, **kwargs): - - expected_search = { - 'sort': sort, - 'sort_metadata': sort_metadata, - 'inheritance_mode': inheritance_mode, - 'inheritance_filter': inheritance_filter or {}, - 'frequencies': frequencies, - 'quality_filter': quality_filter, - 'intervals': intervals, - 'exclude_intervals': exclude_intervals, - 'gene_ids': gene_ids, - 'variant_ids': variant_ids, - 'rs_ids': rs_ids, - } - if annotations: - expected_search['annotations'] = annotations - if annotations_secondary: - expected_search['annotations_secondary'] = annotations_secondary - expected_search.update({field: self.search_model.search[field] for field in search_fields or []}) - - self._test_minimal_search_call(**expected_search, **kwargs) - - @mock.patch('seqr.utils.search.utils.MAX_FAMILY_COUNTS', {'WES': 2, 'WGS': 1}) - @responses.activate - def test_query_variants(self): - variants, total = query_variants(self.results_model, user=self.user) - self.assertListEqual(variants, HAIL_BACKEND_VARIANTS) - self.assertEqual(total, 5) - self.assert_cached_results({'all_results': HAIL_BACKEND_VARIANTS, 'total_results': 5}) - self._test_expected_search_call() - - variants, _ = query_variants( - self.results_model, user=self.user, sort='cadd', skip_genotype_filter=True, page=2, num_results=1, - ) - self.assertListEqual(variants, HAIL_BACKEND_VARIANTS[1:]) - self._test_expected_search_call(sort='cadd', num_results=2) - - raw_variant_locus = '1-10439-AC-A,1-91511686-TCA-G' - self.search_model.search['locus'] = {'rawVariantItems': raw_variant_locus} - query_variants(self.results_model, user=self.user, sort='in_omim') - self._test_expected_search_call( - num_results=2, dataset_type='SNV_INDEL', sample_data={'SNV_INDEL': EXPECTED_SAMPLE_DATA['SNV_INDEL']}, - sort='in_omim', sort_metadata=['ENSG00000240361', 'ENSG00000135953'], - **VARIANT_ID_SEARCH, - ) - - self.search_model.search['locus']['rawVariantItems'] = 'rs1801131' - query_variants(self.results_model, user=self.user, sort='constraint') - self._test_expected_search_call( - sort='constraint', sort_metadata={'ENSG00000223972': 2}, **RSID_SEARCH, - sample_data={'SNV_INDEL': EXPECTED_SAMPLE_DATA['SNV_INDEL'], **EXPECTED_MITO_SAMPLE_DATA}, - ) - - raw_locus = 'CDC7, chr2:1234-5678, chr7:100-10100%10, ENSG00000177000' - self.search_model.search['locus']['rawItems'] = raw_locus - query_variants(self.results_model, user=self.user) - self._test_expected_search_call(**LOCATION_SEARCH, sample_data=EXPECTED_SAMPLE_DATA) - - locus = self.search_model.search.pop('locus') - self.search_model.search['exclude'] = {'rawItems': locus['rawItems']} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call(exclude={}, **EXCLUDE_LOCATION_SEARCH) - - self.search_model.search = { - 'inheritance': {'mode': 'recessive', 'filter': {'affected': { - 'I000004_hg00731': 'N', 'I000005_hg00732': 'A', 'I000006_hg00733': 'U', - }}}, 'annotations': {'frameshift': ['frameshift_variant']}, - } - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - inheritance_mode='recessive', dataset_type='SNV_INDEL', secondary_dataset_type=None, - search_fields=['annotations'], sample_data=CUSTOM_AFFECTED_SAMPLE_DATA, - ) - - self.search_model.search['inheritance']['filter'] = {} - self.search_model.search['annotations_secondary'] = self.search_model.search['annotations'] - sv_annotations = {'structural_consequence': ['LOF']} - self.search_model.search['annotations'] = sv_annotations - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - inheritance_mode='recessive', dataset_type='SV', secondary_dataset_type='SNV_INDEL', - search_fields=['annotations', 'annotations_secondary'], sample_data=EXPECTED_SAMPLE_DATA, - ) - - self.search_model.search['annotations'] = self.search_model.search['annotations_secondary'] - self.search_model.search['annotations_secondary'] = sv_annotations - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - inheritance_mode='recessive', dataset_type='SNV_INDEL', secondary_dataset_type='SV', - search_fields=['annotations', 'annotations_secondary'] - ) - - screen_annotations = {'SCREEN': ['dELS', 'DNase-only']} - self.search_model.search['annotations_secondary'].update(screen_annotations) - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - inheritance_mode='recessive', dataset_type='SNV_INDEL', secondary_dataset_type='ALL', - search_fields=['annotations', 'annotations_secondary'] - ) - - self.search_model.search['annotations_secondary']['structural_consequence'] = [] - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - inheritance_mode='recessive', dataset_type='SNV_INDEL', secondary_dataset_type='SNV_INDEL', - search_fields=['annotations'], annotations_secondary=screen_annotations, omit_data_type='SV_WES', - ) - - self.search_model.search['inheritance']['mode'] = 'x_linked_recessive' - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - inheritance_mode='x_linked_recessive', dataset_type='SNV_INDEL', secondary_dataset_type='SNV_INDEL', - search_fields=['annotations'], annotations_secondary=screen_annotations, sample_data=EXPECTED_SAMPLE_DATA_WITH_SEX, - omit_data_type='SV_WES', - ) - - self.results_model.families.set(Family.objects.filter(id__in=[2, 11, 14])) - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Location must be specified to search across multiple projects') - - self.search_model.search = {'inheritance': {'mode': 'de_novo'}, 'annotations': {'structural_consequence': ['LOF']}} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call(search_fields=['annotations'], dataset_type='SV', call_offset=-2, sample_data={ - 'SV_WES': SV_WES_SAMPLE_DATA, - }) - self._test_expected_search_call(search_fields=['annotations'], dataset_type='SV', call_offset=-1, sample_data={ - 'SV_WGS': SV_WGS_SAMPLE_DATA, - }) - - del self.search_model.search['annotations'] - self.search_model.search['locus'] = {'rawVariantItems': raw_variant_locus} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call(**VARIANT_ID_SEARCH, num_results=2, dataset_type='SNV_INDEL', sample_data={ - 'SNV_INDEL': FAMILY_2_VARIANT_SAMPLE_DATA['SNV_INDEL'] + [FAMILY_11_SAMPLE_WGS], - }) - - self.search_model.search['locus'] = {'rawItems': 'M:10-100 '} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call(intervals=[['M', 10, 100]], sample_data=EXPECTED_MITO_SAMPLE_DATA) - - self.search_model.search['locus']['rawItems'] += raw_locus - variants, total = query_variants(self.results_model, user=self.user) - self.assertListEqual(variants, [ - HAIL_BACKEND_VARIANTS[0], HAIL_BACKEND_VARIANTS[0], HAIL_BACKEND_VARIANTS[1], HAIL_BACKEND_VARIANTS[1], - ]) - self.assertEqual(total, 10) - expected_search = { - 'gene_ids': LOCATION_SEARCH['gene_ids'], - 'intervals': [['M', 10, 100]] + LOCATION_SEARCH['intervals'], - } - self._test_expected_search_call(**expected_search, call_offset=-2, sample_data={ - **FAMILY_2_VARIANT_SAMPLE_DATA, 'SV_WES': SV_WES_SAMPLE_DATA, **EXPECTED_MITO_SAMPLE_DATA, - }) - self._test_expected_search_call(**expected_search, call_offset=-1, sample_data={ - 'SNV_INDEL': [FAMILY_11_SAMPLE_WGS], 'SV_WGS': SV_WGS_SAMPLE_DATA, - }) - - self.search_model.search['locus']['rawItems'] = raw_locus - query_variants(self.results_model, user=self.user) - self._test_expected_search_call(**LOCATION_SEARCH, call_offset=-2, sample_data={ - **FAMILY_2_VARIANT_SAMPLE_DATA, 'SV_WES': SV_WES_SAMPLE_DATA, - }) - self._test_expected_search_call(**LOCATION_SEARCH, call_offset=-1, sample_data={ - 'SNV_INDEL': [FAMILY_11_SAMPLE_WGS], 'SV_WGS': SV_WGS_SAMPLE_DATA, - }) - - self.results_model.families.set(Family.objects.filter(project_id=1)) - query_variants(self.results_model, user=self.user) - self._test_expected_search_call(**LOCATION_SEARCH, sample_data={ - 'SNV_INDEL': FAMILY_1_SAMPLE_DATA['SNV_INDEL'] + EXPECTED_SAMPLE_DATA['SNV_INDEL'], - 'SV_WES': SV_WES_SAMPLE_DATA, - }) - - del self.search_model.search['locus'] - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Location must be specified to search across multiple families in large projects') - - quality_filter = {'min_ab': 10, 'min_gq': 15, 'vcf_filter': 'pass'} - freq_filter = {'callset': {'af': 0.1}, 'gnomad_genomes': {'af': 0.01, 'ac': 3, 'hh': 3}} - genotype_filter = {'genotype': {'I000001_na19675': 'ref_alt'}} - self.search_model.search = deepcopy({ - 'inheritance': {'mode': 'any_affected', 'filter': genotype_filter}, - 'freqs': freq_filter, - 'qualityFilter': quality_filter, - 'in_silico': {'cadd': '11.5', 'sift': 'D'}, - }) - self.results_model.families.set(Family.objects.filter(guid='F000001_1')) - query_variants(self.results_model, user=self.user, sort='prioritized_gene') - expected_freq_filter = {'seqr': freq_filter['callset'], 'gnomad_genomes': freq_filter['gnomad_genomes']} - self._test_expected_search_call( - inheritance_mode=None, inheritance_filter=genotype_filter, sample_data=FAMILY_1_SAMPLE_DATA, - search_fields=['in_silico'], frequencies=expected_freq_filter, quality_filter=quality_filter, - sort='prioritized_gene', sort_metadata={'ENSG00000268903': 1, 'ENSG00000268904': 11}, - ) - - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/search', status=400, body='Bad Search Error') - with self.assertRaises(HTTPError) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(cm.exception.response.status_code, 400) - self.assertEqual(str(cm.exception), 'Bad Search Error') - - @responses.activate - def test_get_variant_query_gene_counts(self): - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/gene_counts', json=GENE_COUNTS, status=200) - - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, GENE_COUNTS) - self.assert_cached_results({'gene_aggs': gene_counts}) - self._test_expected_search_call(url_path='gene_counts', sort=None) - - @responses.activate - def test_variant_lookup(self): - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/lookup', status=200, json=VARIANT_LOOKUP_VARIANT) - variant = variant_lookup(self.user, ('1', 10439, 'AC', 'A'), genome_version='37', foo='bar') - self.assertDictEqual(variant, VARIANT_LOOKUP_VARIANT) - self._test_minimal_search_call(url_path='lookup', expected_search_body={ - 'variant_id': ['1', 10439, 'AC', 'A'], 'genome_version': 'GRCh37', 'foo': 'bar', 'data_type': 'SNV_INDEL', - }) - - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/lookup', status=404) - with self.assertRaises(HTTPError) as cm: - variant_lookup(self.user, ('1', 10439, 'AC', 'A')) - self.assertEqual(cm.exception.response.status_code, 404) - self.assertEqual(str(cm.exception), 'Variant not present in seqr') - self._test_minimal_search_call(url_path='lookup', expected_search_body={ - 'variant_id': ['1', 10439, 'AC', 'A'], 'genome_version': 'GRCh38', 'data_type': 'SNV_INDEL', - }) - - # Test mitochondrial variant lookup - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/lookup', status=400) - with self.assertRaises(InvalidSearchException) as cm: - variant_lookup(self.user, ('M', 11018, 'G', 'T'), genome_version='37') - self.assertEqual(str(cm.exception), 'MITO variants are not available for GRCh37') - - @responses.activate - def test_sv_variant_lookup(self): - sv_families = Family.objects.filter(id__in=[2, 14]) - with self.assertRaises(InvalidSearchException) as cm: - sv_variant_lookup(self.user, 'suffix_140608_DUP', sv_families) - self.assertEqual(str(cm.exception), 'Sample type must be specified to look up a structural variant') - - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/lookup', status=200, json=GCNV_VARIANT4) - variants = sv_variant_lookup(self.user, 'suffix_140608_DUP', sv_families, sample_type='WES') - self.assertListEqual(variants, [GCNV_VARIANT4] + HAIL_BACKEND_VARIANTS) - self._test_minimal_search_call(url_path='lookup', call_offset=-2, expected_search_body={ - 'variant_id': 'suffix_140608_DUP', 'genome_version': 'GRCh38', 'data_type': 'SV_WES', - 'sample_data': ALL_AFFECTED_SAMPLE_DATA['SV_WES'] - }) - self._test_minimal_search_call(expected_search_body={ - 'genome_version': 'GRCh38', 'annotations': {'structural': ['DEL', 'gCNV_DEL']}, - 'padded_interval': {'chrom': '17', 'start': 38721781, 'end': 38735703, 'padding': 0.2}, - 'sample_data': {'SV_WGS': SV_WGS_SAMPLE_DATA}, 'frequencies': None, 'quality_filter': None, - 'sort': None, 'sort_metadata': None, 'num_results': 100, - }) - - # No second lookup call is made for non DELs/DUPs - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/lookup', status=200, json=SV_VARIANT2) - variants = sv_variant_lookup(self.user, 'cohort_2911.chr1.final_cleanup_INS_chr1_160', sv_families, sample_type='WGS') - self._test_minimal_search_call(url_path='lookup', expected_search_body={ - 'variant_id': 'cohort_2911.chr1.final_cleanup_INS_chr1_160', 'genome_version': 'GRCh38', 'data_type': 'SV_WGS', - 'sample_data': SV_WGS_SAMPLE_DATA - }) - self.assertListEqual(variants, [SV_VARIANT2]) - - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/lookup', status=404) - with self.assertRaises(HTTPError) as cm: - sv_variant_lookup(self.user, 'suffix_140608_DUP', sv_families, sample_type='WES') - self.assertEqual(cm.exception.response.status_code, 404) - self.assertEqual(str(cm.exception), 'Variant not present in seqr') - - @responses.activate - def test_get_single_variant(self): - family = self.families.filter(guid='F000002_2').first() - variant = get_single_variant(family, '2-103343353-GAGA-G', user=self.user) - self.assertDictEqual(variant, HAIL_BACKEND_VARIANTS[0]) - self._test_minimal_search_call( - variant_ids=[['2', 103343353, 'GAGA', 'G']], variant_keys=[], - num_results=1, sample_data={'SNV_INDEL': FAMILY_2_VARIANT_SAMPLE_DATA['SNV_INDEL']}) - - get_single_variant(family, 'prefix_19107_DEL', user=self.user) - self._test_minimal_search_call( - variant_ids=[], variant_keys=['prefix_19107_DEL'], - num_results=1, sample_data=EXPECTED_SAMPLE_DATA, omit_data_type='SNV_INDEL') - - get_single_variant(family, 'M-10195-C-A', user=self.user) - self._test_minimal_search_call( - variant_ids=[['M', 10195, 'C', 'A']], variant_keys=[], - num_results=1, sample_data=EXPECTED_MITO_SAMPLE_DATA) - - - get_single_variant(family, '2-103343353-GAGA-G', user=self.user) - self._test_minimal_search_call( - variant_ids=[['2', 103343353, 'GAGA', 'G']], variant_keys=[], - num_results=1, sample_data=FAMILY_2_VARIANT_SAMPLE_DATA) - - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/search', status=200, json={'results': [], 'total': 0}) - with self.assertRaises(InvalidSearchException) as cm: - get_single_variant(family, '10-10334333-A-G', user=self.user) - self.assertEqual(str(cm.exception), 'Variant 10-10334333-A-G not found') - - @responses.activate - def test_get_variants_for_variant_ids(self): - variant_ids = ['2-103343353-GAGA-G', '1-248367227-TC-T', 'prefix-938_DEL'] - get_variants_for_variant_ids(self.families, variant_ids, user=self.user) - expected_sample_data = {k: ALL_AFFECTED_SAMPLE_DATA[k] for k in ['SNV_INDEL', 'SV_WES']} - self._test_minimal_search_call( - variant_ids=[['2', 103343353, 'GAGA', 'G'], ['1', 248367227, 'TC', 'T']], - variant_keys=['prefix-938_DEL'], - num_results=3, sample_data=expected_sample_data) - - del expected_sample_data['SV_WES'] - get_variants_for_variant_ids(self.families, variant_ids, user=self.user, dataset_type='SNV_INDEL') - self._test_minimal_search_call( - variant_ids=[['2', 103343353, 'GAGA', 'G'], ['1', 248367227, 'TC', 'T']], - variant_keys=[], - num_results=2, sample_data=expected_sample_data) From 294df7b512c9384d1b6408f87d030fe715a73509 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 22 Aug 2025 10:00:30 -0400 Subject: [PATCH 12/89] settings cleanup --- settings.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/settings.py b/settings.py index 3762609701..3a0cc4ed3c 100644 --- a/settings.py +++ b/settings.py @@ -374,9 +374,6 @@ KIBANA_ELASTICSEARCH_USER = os.environ.get('KIBANA_ELASTICSEARCH_USER', 'kibana') KIBANA_ELASTICSEARCH_PASSWORD = os.environ.get('KIBANA_ES_PASSWORD') -HAIL_BACKEND_SERVICE_HOSTNAME = os.environ.get('HAIL_BACKEND_SERVICE_HOSTNAME', 'hail-search') -HAIL_BACKEND_SERVICE_PORT = int(os.environ.get('HAIL_BACKEND_SERVICE_PORT', '5000')) - REDIS_SERVICE_HOSTNAME = os.environ.get('REDIS_SERVICE_HOSTNAME', 'localhost') REDIS_SERVICE_PORT = int(os.environ.get('REDIS_SERVICE_PORT', '6379')) From 0e9539cebf194dc5cd80fb45449a154bfab6d152 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 22 Aug 2025 10:12:55 -0400 Subject: [PATCH 13/89] fix unit tests --- seqr/utils/search/search_utils_tests.py | 52 ------------------------- seqr/views/status_tests.py | 14 +++---- 2 files changed, 6 insertions(+), 60 deletions(-) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 6bfacf5cfc..5aaaabd17f 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -648,58 +648,6 @@ def _test_invalid_no_location_search_params(self): pass -@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') -class HailSearchUtilsTests(TestCase, SearchUtilsTests): - databases = ['default', 'reference_data'] - fixtures = ['users', '1kg_project', 'reference_data'] - - def setUp(self): - self.set_up() - - def _assert_expected_search_locus(self, *args, gene_ids=None, exclude_locations=None, parsed_variant_ids=None, variant_ids=None, **kwargs): - super()._assert_expected_search_locus( - *args, gene_ids=None if exclude_locations else gene_ids, variant_ids=parsed_variant_ids, - exclude_intervals=exclude_locations, **kwargs, - ) - - @mock.patch('seqr.utils.search.utils.hail_variant_lookup') - def test_variant_lookup(self, mock_variant_lookup): - super(HailSearchUtilsTests, self).test_variant_lookup(mock_variant_lookup) - - @mock.patch('seqr.utils.search.utils.get_hail_variants') - @mock.patch('seqr.utils.search.utils.hail_variant_lookup') - def test_sv_variant_lookup(self, mock_sv_variant_lookup, mock_get_variants): - super(HailSearchUtilsTests, self).test_sv_variant_lookup(mock_sv_variant_lookup, mock_get_variants) - - @mock.patch('seqr.utils.search.utils.get_hail_variants_for_variant_ids') - def test_get_single_variant(self, mock_call): - mock_call.return_value = [PARSED_VARIANTS[0]] - super(HailSearchUtilsTests, self).test_get_single_variant(mock_call) - - @mock.patch('seqr.utils.search.utils.get_hail_variants_for_variant_ids') - def test_get_variants_for_variant_ids(self, mock_call): - super(HailSearchUtilsTests, self).test_get_variants_for_variant_ids(mock_call) - - @mock.patch('seqr.utils.search.utils.get_hail_variants') - def test_query_variants(self, mock_call): - super(HailSearchUtilsTests, self).test_query_variants(mock_call) - - @mock.patch('seqr.utils.search.utils.get_hail_variants') - def test_get_variant_query_gene_counts(self, mock_call): - super(HailSearchUtilsTests, self).test_get_variant_query_gene_counts(mock_call) - - def test_cached_get_variant_query_gene_counts(self): - super(HailSearchUtilsTests, self).test_cached_get_variant_query_gene_counts() - - self.set_cache({'all_results': [PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT] + [SV_VARIANT1], 'total_results': 2}) - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 2, 'families': {'F000003_3': 2, 'F000011_11': 2}}, - 'ENSG00000228198': {'total': 2, 'families': {'F000003_3': 2, 'F000011_11': 2}}, - 'ENSG00000171621': {'total': 1, 'families': {'F000011_11': 1}}, - }) - - @mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', 'testhost') class ClickhouseSearchUtilsTests(DifferentDbTransactionSupportMixin, TestCase, SearchUtilsTests): databases = '__all__' diff --git a/seqr/views/status_tests.py b/seqr/views/status_tests.py index f96464d0ca..2746a49f3b 100644 --- a/seqr/views/status_tests.py +++ b/seqr/views/status_tests.py @@ -7,7 +7,6 @@ from seqr.utils.search.elasticsearch.es_utils_tests import urllib3_responses -@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') class StatusTest(object): def _test_status_error(self, url, mock_logger): @@ -21,8 +20,9 @@ def _test_status_error(self, url, mock_logger): mock.call('Database "clickhouse_write" connection error: No connection'), mock.call('Database "clickhouse" connection error: No connection'), mock.call('Redis connection error: Bad connection'), - mock.call(f'Search backend connection error: {self.SEARCH_BACKEND_ERROR}'), ] + if self.SEARCH_BACKEND_ERROR: + calls.append(mock.call(f'Search backend connection error: {self.SEARCH_BACKEND_ERROR}')) if self.HAS_KIBANA: calls.append(mock.call('Search Admin connection error: Kibana Error 400: Bad Request')) mock_logger.error.assert_has_calls(calls) @@ -81,16 +81,14 @@ def _assert_expected_requests(self): self.assertListEqual([call.request.url for call in urllib3_responses.calls], ['/', '/status', '/', '/status']) -class HailSearchStatusTest(TestCase, StatusTest): +class ClickhouseStatusTest(TestCase, StatusTest): - SEARCH_BACKEND_ERROR = '400: Bad Request' + SEARCH_BACKEND_ERROR = None HAS_KIBANA = False @mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', '') - @mock.patch('seqr.utils.search.hail_search_utils.HAIL_BACKEND_SERVICE_HOSTNAME', 'test-hail') def test_status(self, *args): - super(HailSearchStatusTest, self).test_status(*args) + super().test_status(*args) def _assert_expected_requests(self): - self.assertEqual(len(urllib3_responses.calls), 1) - self.assertEqual(urllib3_responses.calls[0].request.url, '/status') + self.assertEqual(len(urllib3_responses.calls), 0) From 7f7662d0d4fd552ba02b3555e063422788004da8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 22 Aug 2025 18:13:25 -0400 Subject: [PATCH 14/89] clean up github workflows --- .../workflows/dev-hail-search-release.yaml | 64 --------------- .github/workflows/docker-lint.yaml | 5 -- .github/workflows/hail-search-unit-tests.yaml | 38 --------- .../workflows/prod-hail-search-release.yaml | 79 ------------------- .github/workflows/unit-tests.yml | 6 +- 5 files changed, 1 insertion(+), 191 deletions(-) delete mode 100644 .github/workflows/dev-hail-search-release.yaml delete mode 100644 .github/workflows/hail-search-unit-tests.yaml delete mode 100644 .github/workflows/prod-hail-search-release.yaml diff --git a/.github/workflows/dev-hail-search-release.yaml b/.github/workflows/dev-hail-search-release.yaml deleted file mode 100644 index afd197cf39..0000000000 --- a/.github/workflows/dev-hail-search-release.yaml +++ /dev/null @@ -1,64 +0,0 @@ -name: seqr hail search dev release -on: - workflow_run: - workflows: ["Hail Search Unit Tests"] - types: - - completed - branches: - - dev - -permissions: - id-token: write - -jobs: - docker: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - name: checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.event.workflow_run.head_branch }} - - - name: authenticate to google cloud - id: "auth" - uses: google-github-actions/auth@v0 - with: - workload_identity_provider: "${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}" - service_account: "${{ secrets.RUN_SA_EMAIL }}" - - - name: "setup gcloud sdk" - uses: google-github-actions/setup-gcloud@v0 - - - name: Build and push images - run: |- - gcloud builds submit --quiet --substitutions="COMMIT_SHA=${{ github.event.workflow_run.head_sha }},_CUSTOM_BRANCH_TAG=gcloud-dev" --config .cloudbuild/seqr-hail-search-docker.cloudbuild.yaml --gcs-log-dir=gs://seqr-github-actions-logs/logs . - - helm_update: - runs-on: ubuntu-latest - needs: docker - steps: - - name: Retrieve tgg-helm repo for broad seqr chart - uses: actions/checkout@v3 - with: - repository: broadinstitute/tgg-helm - token: ${{ secrets.SEQR_VERSION_UPDATE_TOKEN }} - ref: main - persist-credentials: false - fetch-depth: 0 - - - name: update image tag in the dev broad seqr chart - uses: mikefarah/yq@v4.22.1 - with: - cmd: > - yq -i '.seqr-platform.hail-search.image.tag = "${{ github.event.workflow_run.head_sha }}"' charts/dev-broad-seqr/values.yaml - - - name: Commit and Push changes - uses: Andro999b/push@v1.3 - with: - repository: broadinstitute/tgg-helm - branch: main - github_token: ${{ secrets.SEQR_VERSION_UPDATE_TOKEN }} - author_email: ${{ github.actor }}@users.noreply.github.com - author_name: tgg-automation - message: "Update hail-search dev release docker tag to ${{ github.event.workflow_run.head_sha }}" diff --git a/.github/workflows/docker-lint.yaml b/.github/workflows/docker-lint.yaml index b928c7c161..67a481e784 100644 --- a/.github/workflows/docker-lint.yaml +++ b/.github/workflows/docker-lint.yaml @@ -9,7 +9,6 @@ on: - master paths: - deploy/docker/seqr/Dockerfile - - hail_search/deploy/Dockerfile - vlm/deploy/Dockerfile - .hadolint.yaml - .docker-compose.yaml @@ -21,7 +20,6 @@ on: - master paths: - deploy/docker/seqr/Dockerfile - - hail_search/deploy/Dockerfile - vlm/deploy/Dockerfile - .hadolint.yaml - .docker-compose.yaml @@ -37,9 +35,6 @@ jobs: - uses: hadolint/hadolint-action@v1.5.0 with: dockerfile: deploy/docker/seqr/Dockerfile - - uses: hadolint/hadolint-action@v1.5.0 - with: - dockerfile: hail_search/deploy/Dockerfile - uses: hadolint/hadolint-action@v1.5.0 with: dockerfile: vlm/deploy/Dockerfile diff --git a/.github/workflows/hail-search-unit-tests.yaml b/.github/workflows/hail-search-unit-tests.yaml deleted file mode 100644 index 389afdc3e7..0000000000 --- a/.github/workflows/hail-search-unit-tests.yaml +++ /dev/null @@ -1,38 +0,0 @@ -name: Hail Search Unit Tests - -# Run the test suite on pushes (incl. merges) to master and dev -# Run the test suite when a PR is opened, pushed to, or reopened -on: - push: - branches: - - master - - dev - paths: - - 'hail_search/**' - pull_request: - types: [opened, synchronize, reopened] - paths: - - 'hail_search/**' - -jobs: - hail_search: - runs-on: ubuntu-latest - container: hailgenetics/hail:0.2.128 - - steps: - - uses: actions/checkout@v2 - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip wheel - pip install -r hail_search/requirements-test.txt - - name: Run coverage tests - run: | - export HAIL_SEARCH_DATA_DIR=./hail_search/fixtures - export REFERENCE_DATASETS_DIR=./hail_search/fixtures/reference_data - export MAX_GENE_INTERVALS=3 - export MACHINE_MEM=24 - export JAVA_OPTS_XSS=16M - coverage run --source="./hail_search" --omit="./hail_search/__main__.py","./hail_search/test_utils.py" -m pytest hail_search/ - coverage report --fail-under=99 - coverage report | (! grep [1-8][0-9]%) - coverage report | (! grep 9[0-5]%) diff --git a/.github/workflows/prod-hail-search-release.yaml b/.github/workflows/prod-hail-search-release.yaml deleted file mode 100644 index 0a2dbb0ee7..0000000000 --- a/.github/workflows/prod-hail-search-release.yaml +++ /dev/null @@ -1,79 +0,0 @@ -name: seqr hail search prod release -on: - workflow_run: - workflows: ["Hail Search Unit Tests"] - types: - - completed - branches: - - master - -permissions: - id-token: write - -jobs: - docker: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - name: checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.event.workflow_run.head_branch }} - - - name: authenticate to google cloud - id: "auth" - uses: google-github-actions/auth@v0 - with: - workload_identity_provider: "${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}" - service_account: "${{ secrets.RUN_SA_EMAIL }}" - - - name: "setup gcloud sdk" - uses: google-github-actions/setup-gcloud@v0 - - - name: Build and push images - run: |- - gcloud builds submit --quiet --substitutions="COMMIT_SHA=${{ github.event.workflow_run.head_sha }},_CUSTOM_BRANCH_TAG=gcloud-prod" --config .cloudbuild/seqr-hail-search-docker.cloudbuild.yaml --gcs-log-dir=gs://seqr-github-actions-logs/logs . - - helm_update: - runs-on: ubuntu-latest - needs: docker - steps: - - uses: actions/checkout@v3 - with: - repository: broadinstitute/seqr-helm - ref: main - persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token - fetch-depth: 0 # otherwise, you will failed to push refs to dest repo - - - name: Get latest hail-search version - uses: mikefarah/yq@v4.22.1 - id: current - with: - cmd: > - yq -r '.version' charts/hail-search/Chart.yaml - - - name: Bump version - id: bump - uses: cbrgm/semver-bump-action@main - with: - current-version: ${{ steps.current.outputs.result }} - bump-level: minor - - - name: Update appVersion and version in seqr Chart file - uses: mikefarah/yq@v4.22.1 - with: - cmd: > - yq -i ' - .appVersion = "${{ github.event.workflow_run.head_sha }}" | - .version = "${{ steps.bump.outputs.new_version }}" - ' charts/hail-search/Chart.yaml - - - name: Commit and Push changes - uses: Andro999b/push@v1.3 - with: - repository: broadinstitute/seqr-helm - branch: main - github_token: ${{ secrets.SEQR_VERSION_UPDATE_TOKEN }} - author_email: ${{ github.actor }}@users.noreply.github.com - author_name: tgg-automation - message: "Update hail-search chart appVersion to ${{ github.event.workflow_run.head_sha }}" diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6a8e929bfa..c8d1a2996a 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -8,16 +8,12 @@ on: - master - dev paths-ignore: - - 'hail_search/**' - 'vlm/**' - - '.github/workflows/*hail-search*.yaml' - '.github/workflows/*vlm*.yaml' pull_request: types: [opened, synchronize, reopened] paths-ignore: - - 'hail_search/**' - 'vlm/**' - - '.github/workflows/*hail-search*.yaml' - '.github/workflows/*vlm*.yaml' jobs: @@ -73,7 +69,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v1 # TODO remove? with: java-version: "8" # The JDK version needed by hail java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk From c382fb2b451eddd657a666d6cb366874beaf6144 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 22 Aug 2025 18:16:12 -0400 Subject: [PATCH 15/89] remove vlm hail backend --- .github/workflows/vlm-unit-tests.yaml | 26 ++------------------------ vlm/hail_backend_utils.py | 15 --------------- vlm/match.py | 7 ++----- 3 files changed, 4 insertions(+), 44 deletions(-) delete mode 100644 vlm/hail_backend_utils.py diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 35c1c631e8..f8dcf4d23b 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -12,38 +12,16 @@ on: paths: - 'vlm/**' - '.github/workflows/*vlm*.yaml' - - 'hail_search/fixtures/*' pull_request: types: [opened, synchronize, reopened] paths: - 'vlm/**' - '.github/workflows/*vlm*.yaml' - - 'hail_search/fixtures/*' jobs: - vlm_hail_backend: - runs-on: ubuntu-latest - container: hailgenetics/hail:0.2.128 - - steps: - - uses: actions/checkout@v2 - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip wheel - pip install -r vlm/requirements-test.txt - - name: Run coverage tests - run: | - export VLM_DATA_DIR=./hail_search/fixtures - export SEQR_BASE_URL=https://test-seqr.org/ - export NODE_ID=TestVLM - export MACHINE_MEM=24 - export JAVA_OPTS_XSS=16M - coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py","./vlm/clickhouse_utils.py" -m pytest vlm/ - coverage report --fail-under=95 - vlm_clickhouse: runs-on: ubuntu-latest - container: hailgenetics/hail:0.2.128 + container: hailgenetics/hail:0.2.128 # TODO different image? services: clickhouse: @@ -78,6 +56,6 @@ jobs: export CLICKHOUSE_VLM_USERNAME=vlm_test_user export CLICKHOUSE_VLM_PASSWORD=vlm_test_password export CLICKHOUSE_DATABASE=test_seqr - coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py","./vlm/hail_backend_utils.py" -m pytest vlm/ + coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py" -m pytest vlm/ coverage report --fail-under=95 diff --git a/vlm/hail_backend_utils.py b/vlm/hail_backend_utils.py deleted file mode 100644 index 6d12287870..0000000000 --- a/vlm/hail_backend_utils.py +++ /dev/null @@ -1,15 +0,0 @@ -import hail as hl -import os - -VLM_DATA_DIR = os.environ.get('VLM_DATA_DIR') - - -def get_hail_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, genome_build: str) -> hl.Struct: - interval = hl.eval(hl.interval(locus, locus, includes_start=True, includes_end=True)) - ht = hl.read_table( - f'{VLM_DATA_DIR}/{genome_build}/SNV_INDEL/annotations.ht', _intervals=[interval], _filter_intervals=True, - ) - ht = ht.filter(ht.alleles == hl.array([ref, alt])) - - counts = ht.aggregate(hl.agg.take(ht.gt_stats, 1)) - return (counts[0].AC, counts[0].hom) if counts else (0, 0) diff --git a/vlm/match.py b/vlm/match.py index 0795e0f87b..8c2b42cd62 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -3,9 +3,7 @@ import os from vlm.clickhouse_utils import get_clickhouse_variant_counts -from vlm.hail_backend_utils import get_hail_variant_counts -CLICKHOUSE_ENABLED = bool(os.environ.get('CLICKHOUSE_SERVICE_HOSTNAME')) SEQR_BASE_URL = os.environ.get('SEQR_BASE_URL') VLM_DEFAULT_CONTACT_EMAIL = os.environ.get('VLM_DEFAULT_CONTACT_EMAIL') NODE_ID = os.environ.get('NODE_ID') @@ -42,12 +40,11 @@ def get_variant_match(query: dict) -> dict: chrom, pos, ref, alt, genome_build = _parse_match_query(query) locus = hl.locus(chrom, pos, reference_genome=genome_build) - get_counts_func = get_clickhouse_variant_counts if CLICKHOUSE_ENABLED else get_hail_variant_counts - ac, hom = get_counts_func(locus, ref, alt, genome_build) + ac, hom = get_clickhouse_variant_counts(locus, ref, alt, genome_build) liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 liftover_locus = hl.liftover(locus, liftover_genome_build) - lift_ac, lift_hom = get_counts_func(liftover_locus, ref, alt, liftover_genome_build) + lift_ac, lift_hom = get_clickhouse_variant_counts(liftover_locus, ref, alt, liftover_genome_build) url = _get_contact_url( chrom, pos, ref, alt, genome_build, liftover_genome_build, liftover_locus if lift_ac and not ac else None, From bd6fe237560db7dc614974e639c9fdb941745d65 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 25 Aug 2025 16:56:15 -0400 Subject: [PATCH 16/89] fix local check samples test --- .../check_for_new_samples_from_pipeline.py | 2 - ...eck_for_new_samples_from_pipeline_tests.py | 244 ++++-------------- 2 files changed, 52 insertions(+), 194 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index f980c72026..3c727437e6 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -37,8 +37,6 @@ Sample.DATASET_TYPE_SV_CALLS: f'{Sample.DATASET_TYPE_SV_CALLS}_WGS', } USER_EMAIL = 'manage_command' -MAX_LOOKUP_VARIANTS = 1000 -MAX_RELOAD_VARIANTS = 100000 RELATEDNESS_CHECK_NAME = 'relatedness_check' PDO_COPY_FIELDS = [ diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index d46c613e38..08aa98f6d4 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -6,14 +6,12 @@ import mock import responses -from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, AuthenticationTestCase +from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, AuthenticationTestCase, DifferentDbTransactionSupportMixin from seqr.models import Project, Family, Individual, Sample, SavedVariant SEQR_URL = 'https://seqr.broadinstitute.org/' PROJECT_GUID = 'R0003_test' EXTERNAL_PROJECT_GUID = 'R0004_non_analyst_project' -MOCK_HAIL_HOST = 'test-hail-host' -MOCK_HAIL_ORIGIN = f'http://{MOCK_HAIL_HOST}' GUID_ID = 54321 GCNV_GUID_ID = 12345 @@ -161,12 +159,12 @@ '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/manual__2025-01-13/_ERRORS_REPORTED', '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/manual__2025-01-13/validation_errors.json', '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/manual__2025-01-14/validation_errors.json', - '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/auto__2023-08-09/_SUCCESS', - '/seqr/seqr-hail-search-data/GRCh37/SNV_INDEL/runs/manual__2023-11-02/_SUCCESS', - '/seqr/seqr-hail-search-data/GRCh38/MITO/runs/auto__2024-08-12/_SUCCESS', - '/seqr/seqr-hail-search-data/GRCh38/GCNV/runs/auto__2024-09-14/_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/auto__2023-08-09/_CLICKHOUSE_LOAD_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh37/SNV_INDEL/runs/manual__2023-11-02/_CLICKHOUSE_LOAD_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh38/MITO/runs/auto__2024-08-12/_CLICKHOUSE_LOAD_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh38/GCNV/runs/auto__2024-09-14/_CLICKHOUSE_LOAD_SUCCESS', '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/manual__2025-01-24/validation_errors.json', - '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/hail_search_to_clickhouse_migration_WGS_R0877_neptune/_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/hail_search_to_clickhouse_migration_WGS_R0877_neptune/_CLICKHOUSE_LOAD_SUCCESS', ] RUN_PATHS = [ b'gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-13/', @@ -375,10 +373,9 @@ def set_up(self): self.addCleanup(patcher.stop) Sample.objects.filter(guid=OLD_DATA_SAMPLE_GUID).update(sample_type='WES') - def _test_call(self, error_logs=None, reload_annotations_logs=None, run_loading_logs=None, reload_calls=None, num_runs=5, has_reload_calls=False): + def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=5): self._set_loading_files() self.reset_logs() - responses.calls.reset() call_command('check_for_new_samples_from_pipeline') @@ -407,51 +404,23 @@ def _test_call(self, error_logs=None, reload_annotations_logs=None, run_loading_ {'severity': 'ERROR', '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent'}, )) logs.append(('Reset 2 cached results', None)) - logs += reload_annotations_logs or [] logs += [] if single_call else [(log, None) for log in self.VALIDATION_LOGS] logs.append(('DONE', None)) + self.maxDiff = None self.assert_json_logs(user=None, expected=logs) self.mock_redis.return_value.delete.assert_called_with('search_results__*', 'variant_lookup_results__*') - # Test reload saved variants - num_airtable_loading_calls, num_airtable_validation_calls = self._assert_expected_airtable_calls(has_reload_calls, single_call) - if not reload_calls: - self.assertEqual(len(responses.calls), num_airtable_loading_calls + num_airtable_validation_calls) - return - - reload_annotation_calls = [] if single_call else [ - {'genome_version': 'GRCh38', 'data_type': 'SNV_INDEL', 'variant_ids': [['1', 1562437, 'G', 'CA']]}, - {'genome_version': 'GRCh38', 'data_type': 'SNV_INDEL', 'variant_ids': [['1', 1562437, 'G', 'CA']]}, - {'genome_version': 'GRCh38', 'data_type': 'SNV_INDEL', 'variant_ids': [['1', 46859832, 'G', 'A']]}, - {'genome_version': 'GRCh38', 'data_type': 'SV_WES', 'variant_ids': ['prefix_19107_DEL']} - ] - self.assertEqual(len(responses.calls), len(reload_calls) +len(reload_annotation_calls) + num_airtable_loading_calls + num_airtable_validation_calls) - for i, call in enumerate(reload_calls or []): - resp = responses.calls[i+num_airtable_loading_calls] - self.assertEqual(resp.request.url, f'{MOCK_HAIL_ORIGIN}:5000/search') - self.assertEqual(resp.request.headers.get('From'), 'manage_command') - self.assertDictEqual(json.loads(resp.request.body), call) - - for i, body in enumerate(reload_annotation_calls): - multi_lookup_request = responses.calls[num_airtable_loading_calls+len(reload_calls)+i].request - self.assertEqual(multi_lookup_request.url, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup') - self.assertEqual(multi_lookup_request.headers.get('From'), 'manage_command') - self.assertDictEqual(json.loads(multi_lookup_request.body), body) + self._assert_expected_airtable_calls(bool(run_loading_logs), single_call) def _additional_loading_logs(self, data_type, version): return [] - @mock.patch('seqr.management.commands.check_for_new_samples_from_pipeline.MAX_LOOKUP_VARIANTS', 1) @mock.patch('seqr.views.utils.airtable_utils.BASE_URL', 'https://test-seqr.org/') @mock.patch('seqr.views.utils.airtable_utils.MAX_UPDATE_RECORDS', 2) - @mock.patch('seqr.management.commands.check_for_new_samples_from_pipeline.MAX_RELOAD_VARIANTS') @mock.patch('seqr.views.utils.export_utils.TemporaryDirectory') @mock.patch('seqr.utils.communication_utils.EmailMultiAlternatives') - @responses.activate - def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): - mock_max_reload_variants.__lt__.return_value = False - + def test_command(self, mock_email, mock_temp_dir): # Test errors self._set_empty_loading_files() with self.assertRaises(CommandError) as ce: @@ -485,69 +454,33 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): # Test success self.mock_send_slack.reset_mock() mock_email.reset_mock() - search_body = { - 'genome_version': 'GRCh38', 'num_results': 1, 'variant_ids': [['1', 248367227, 'TC', 'T']], 'variant_keys': [], - } - reload_snv_indel_calls = [ - {**search_body, 'sample_data': {'SNV_INDEL': [ - {'individual_guid': 'I000016_na20888', 'family_guid': 'F000012_12', 'project_guid': 'R0003_test', 'affected': 'A', 'sample_id': 'NA20888', 'sample_type': 'WES'}, - {'individual_guid': 'I000017_na20889', 'family_guid': 'F000012_12', 'project_guid': 'R0003_test', 'affected': 'A', 'sample_id': 'NA20889', 'sample_type': 'WES'}, - ]}}, - {**search_body, 'sample_data': {'SNV_INDEL': [ - {'individual_guid': 'I000018_na21234', 'family_guid': 'F000014_14', 'project_guid': 'R0004_non_analyst_project', 'affected': 'A', 'sample_id': 'NA21234', 'sample_type': 'WES'}, - ]}}, - ] - reload_fetched_annotations_logs = [ - ('Reloading shared annotations for 1 fetched SNV_INDEL GRCh38 saved variants', None), - ('update 1 SavedVariants', {'dbUpdate': { - 'dbEntity': 'SavedVariant', - 'entityIds': ['SV0000002_1248367227_r0390_100'], - 'updateFields': ['saved_variant_json'], - 'updateType': 'bulk_update'}, - }), - ] create_snv_indel_samples_logs = [ ('Loading 4 WES SNV_INDEL samples in 2 projects', None), ('create 4 Samples', {'dbUpdate': mock.ANY}), ('update 4 Samples', {'dbUpdate': mock.ANY}), ] - update_sample_qc_logs = [ + update_sample_logs = [ ('update 2 Individuals', {'dbUpdate': { 'dbEntity': 'Individual', 'entityIds': ['I000001_na19675', 'I000015_na20885'], 'updateFields': ['filter_flags', 'pop_platform_filters', 'population'], 'updateType': 'bulk_update'}} - ), + ), + ('Reloading saved variants in 2 projects', None), + ('Reloading genotypes for 0 SNV_INDEL variants in family F000012_12', None), + ('Updated 0 variants in 2 families for project Test Reprocessed Project', None), + ('Reloading genotypes for 1 SNV_INDEL variants in family F000014_14', None), + ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), + ('Updated 1 variants in 1 families for project Non-Analyst Project', None), + ('Reload Summary: ', None), + (' Non-Analyst Project: Updated 1 variants', None), ] - self._test_call(reload_calls=reload_snv_indel_calls + [ - {'genome_version': 'GRCh38', 'num_results': 1, 'variant_ids': [], 'variant_keys': ['prefix_19107_DEL'], 'sample_data': {'SV_WES': [ - {'individual_guid': 'I000017_na20889', 'family_guid': 'F000012_12', 'project_guid': 'R0003_test', 'affected': 'A', 'sample_id': 'NA20889', 'sample_type': 'WES'}, - ]}}, - ], reload_annotations_logs=reload_fetched_annotations_logs + [ - ('Reloading shared annotations for 2 SNV_INDEL GRCh38 saved variants in chromosome 1 (2 unique)', None), - ('Fetched 1 additional variants in chromosome 1', None), - ('update 1 SavedVariants', {'dbUpdate': { - 'dbEntity': 'SavedVariant', - 'entityIds': ['SV0059956_11560662_f019313_1'], - 'updateFields': ['saved_variant_json'], - 'updateType': 'bulk_update'}, - }), - ('Fetched 1 additional variants in chromosome 1', None), - ('update 1 SavedVariants', {'dbUpdate': { - 'dbEntity': 'SavedVariant', - 'entityIds': ['SV0059956_11560662_f019313_1'], - 'updateFields': ['saved_variant_json'], - 'updateType': 'bulk_update'}, - }), - ] + [(f'No additional SNV_INDEL GRCh38 saved variants to update in chromosome {chrom}', None) for chrom in ['2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', 'X', 'Y', 'M']] + [ - ('Reloading shared annotations for 1 SV_WES GRCh38 saved variants (1 unique)', None), - ('Fetched 1 additional variants', None), - ], run_loading_logs={ + self._test_call(run_loading_logs={ 'GRCh38/SNV_INDEL': create_snv_indel_samples_logs + [ ('update 1 Samples', {'dbUpdate': mock.ANY}), ('update 2 Familys', {'dbUpdate': mock.ANY}), ] + self.AIRTABLE_LOGS + [ ('update 3 Familys', {'dbUpdate': mock.ANY}), - ] + update_sample_qc_logs + self.RELOAD_PROJECT_VARIANT_LOGS, + ] + update_sample_logs, 'GRCh38/MITO': [ ('Loading 2 WGS MITO samples in 1 projects', None) ], @@ -558,7 +491,10 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): ('update 3 Samples', {'dbUpdate': mock.ANY}), ('update 1 Familys', {'dbUpdate': mock.ANY}), ('Reloading saved variants in 2 projects', None), - ] + self.RELOAD_SV_VARIANT_LOGS, + ('Updated 0 variants in 1 families for project 1kg project nåme with uniçøde', None), + ('Updated 0 variants in 1 families for project Test Reprocessed Project', None), + ('Reload Summary: ', None), + ], }, error_logs={ 'manual__2023-11-02': 'Invalid families in run metadata GRCh37/SNV_INDEL: manual__2023-11-02 - F0000123_ABC', 'auto__2024-08-12': 'Matches not found for sample ids: NA20885, NA22882', @@ -650,7 +586,11 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): ) self.assertEqual(Family.objects.get(guid='F000014_14').analysis_status, 'Rncc') - self._assert_saved_variant_models_updated() + saved_variant = SavedVariant.objects.get(key=100, family_id=14) + self.assertDictEqual(saved_variant.genotypes, {'I000018_na21234': { + 'ab': 0.0, 'dp': 49, 'gq': 99, 'numAlt': 2, 'filters': [], + 'sampleId': 'NA21234', 'familyGuid': 'F000014_14', 'sampleType': 'WGS', 'individualGuid': 'I000018_na21234', + }}) # Test notifications self.assertEqual(self.mock_send_slack.call_count, 7 + len(self.ADDITIONAL_SLACK_CALLS)) @@ -741,20 +681,18 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): self.mock_redis.return_value.delete.assert_not_called() # Test reloading shared annotations is skipped if too many saved variants - mock_max_reload_variants.__lt__.return_value = True snv_indel_samples.delete() airtable_logs = self.AIRTABLE_LOGS[:-1] if self.AIRTABLE_LOGS: airtable_logs.append(('Fetched 1 AnVIL Seqr Loading Requests Tracking records from airtable', None)) - self._test_call(num_runs=2, reload_annotations_logs=reload_fetched_annotations_logs + [ - ('Skipped reloading all 2 saved variant annotations for SNV_INDEL GRCh38', None), - ], reload_calls=reload_snv_indel_calls, run_loading_logs={ - 'GRCh38/SNV_INDEL': create_snv_indel_samples_logs + airtable_logs + update_sample_qc_logs + self.RELOAD_PROJECT_VARIANT_LOGS, + self._test_call(num_runs=2, run_loading_logs={ + 'GRCh38/SNV_INDEL': create_snv_indel_samples_logs + airtable_logs + update_sample_logs, }) -@mock.patch('seqr.utils.search.hail_search_utils.HAIL_BACKEND_SERVICE_HOSTNAME', MOCK_HAIL_HOST) -class LocalCheckNewSamplesTest(AuthenticationTestCase, CheckNewSamplesTest): - fixtures = ['users', '1kg_project'] + +class LocalCheckNewSamplesTest(DifferentDbTransactionSupportMixin, AuthenticationTestCase, CheckNewSamplesTest): + fixtures = ['users', '1kg_project', 'clickhouse_saved_variants'] + databases = '__all__' ES_HOSTNAME = '' @@ -766,22 +704,6 @@ class LocalCheckNewSamplesTest(AuthenticationTestCase, CheckNewSamplesTest): LIST_FILE_LOGS = [] AIRTABLE_LOGS = [] VALIDATION_LOGS = [] - RELOAD_PROJECT_VARIANT_LOGS = [ - ('Reloading saved variants in 2 projects', None), - ('Updated 0 variants in 2 families for project Test Reprocessed Project', None), - ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), - ('Updated 1 variants in 1 families for project Non-Analyst Project', None), - ('Reload Summary: ', None), - (' Non-Analyst Project: Updated 1 variants', None), - ] - RELOAD_SV_VARIANT_LOGS = [ - (mock.ANY, {'severity': 'ERROR', '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent'}), - ('Error reloading variants in Test Reprocessed Project: Bad Request', {'severity': 'ERROR','@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent'}), - ('Reload Summary: ', None), - ('Skipped the following 1 project with no saved variants: 1kg project nåme with uniçøde', None), - ('1 failed projects', None), - (' Test Reprocessed Project: Bad Request', None), - ] ADDITIONAL_SLACK_CALLS = [ mock.call( 'seqr-data-loading', @@ -809,26 +731,6 @@ def setUp(self): self.set_up() super().setUp() - def test_command(self, *args, **kwargs): - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=200, json={ - 'results': [{'variantId': '1-248367227-TC-T', 'familyGuids': ['F000014_14'], 'updated_field': 'updated_value'}], - 'total': 1, - }) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup', status=400) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup', status=200, json={ - 'results': [{'variantId': '1-46859832-G-A', 'updated_new_field': 'updated_value', 'rsid': 'rs123'}], - }) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=200, json={ - 'results': [{'variantId': '1-248367227-TC-T', 'familyGuids': ['F000014_14'], 'updated_field': 'updated_value'}], - 'total': 1, - }) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=400) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=200, json={ - 'results': [{'variantId': '1-248367227-TC-T', 'familyGuids': ['F000014_14'], 'updated_field': 'updated_value'}], - 'total': 1, - }) - super().test_command(*args, **kwargs) - def _set_empty_loading_files(self): self.mock_glob.return_value = [] @@ -850,7 +752,7 @@ def _assert_expected_loading_file_calls(self, single_call): self.mock_glob.assert_called_with('/seqr/seqr-hail-search-data/*/*/runs/*/*', recursive=False) self.mock_open.assert_has_calls([ mock.call(LOCAL_RUN_PATHS[2], 'r'), - *[mock.call(path.replace('_SUCCESS', 'metadata.json'), 'r') for path in LOCAL_RUN_PATHS[3:-1]] + *[mock.call(path.replace('_CLICKHOUSE_LOAD_SUCCESS', 'metadata.json'), 'r') for path in LOCAL_RUN_PATHS[3:-1]] ], any_order=True) self.assertEqual(self.mock_mkdir.call_count, 0 if single_call else 2) self.assertEqual(list(self.mock_written_files.keys()), [ @@ -859,28 +761,7 @@ def _assert_expected_loading_file_calls(self, single_call): ]) def _assert_expected_airtable_calls(self, *args, **kwargs): - return 0, 0 - - def _assert_saved_variant_models_updated(self): - updated_variants = SavedVariant.objects.filter(saved_variant_json__updated_field='updated_value') - self.assertEqual(len(updated_variants), 2) - self.assertSetEqual( - {v.guid for v in updated_variants}, - {'SV0000006_1248367227_r0004_non', 'SV0000002_1248367227_r0390_100'} - ) - reloaded_variant = next(v for v in updated_variants if v.guid == 'SV0000006_1248367227_r0004_non') - annotation_updated_variant = next(v for v in updated_variants if v.guid == 'SV0000002_1248367227_r0390_100') - self.assertEqual(len(reloaded_variant.saved_variant_json), 3) - self.assertListEqual(reloaded_variant.saved_variant_json['familyGuids'], ['F000014_14']) - self.assertEqual(len(annotation_updated_variant.saved_variant_json), 17) - self.assertListEqual(annotation_updated_variant.saved_variant_json['familyGuids'], ['F000001_1']) - - annotation_updated_json = SavedVariant.objects.get(guid='SV0059956_11560662_f019313_1').saved_variant_json - self.assertEqual(len(annotation_updated_json), 16) - self.assertEqual(annotation_updated_json['updated_new_field'], 'updated_value') - self.assertEqual(annotation_updated_json['rsid'], 'rs123') - self.assertEqual(annotation_updated_json['mainTranscriptId'], 'ENST00000505820') - self.assertEqual(len(annotation_updated_json['genotypes']), 3) + pass class AirtableCheckNewSamplesTest(AnvilAuthenticationTestCase, CheckNewSamplesTest): @@ -925,21 +806,6 @@ class AirtableCheckNewSamplesTest(AnvilAuthenticationTestCase, CheckNewSamplesTe '==> gsutil cat gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-24/validation_errors.json', '==> gsutil mv /mock/tmp/* gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-24/', ] - RELOAD_PROJECT_VARIANT_LOGS = [ - ('Reloading saved variants in 2 projects', None), - ('Reloading genotypes for 0 SNV_INDEL variants in family F000012_12', None), - ('Updated 0 variants in 2 families for project Test Reprocessed Project', None), - ('Reloading genotypes for 1 SNV_INDEL variants in family F000014_14', None), - ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), - ('Updated 1 variants in 1 families for project Non-Analyst Project', None), - ('Reload Summary: ', None), - (' Non-Analyst Project: Updated 1 variants', None), - ] - RELOAD_SV_VARIANT_LOGS = [ - ('Updated 0 variants in 1 families for project 1kg project nåme with uniçøde', None), - ('Updated 0 variants in 1 families for project Test Reprocessed Project', None), - ('Reload Summary: ', None), - ] ADDITIONAL_SLACK_CALLS = [ mock.call( 'anvil-data-loading', @@ -977,6 +843,7 @@ def setUp(self): self.set_up() super().setUp() + @responses.activate def test_command(self, *args, **kwargs): responses.add( responses.GET, @@ -1009,10 +876,6 @@ def test_command(self, *args, **kwargs): }))) super().test_command(*args, **kwargs) - def _test_call(self, *args, reload_annotations_logs=None, reload_calls=None, **kwargs): - # No reloading for clickhouse - super()._test_call(*args, has_reload_calls=bool(reload_calls), **kwargs) - def _set_empty_loading_files(self): self.mock_subprocess.return_value.communicate.return_value = b'', b'One or more URLs matched no objects' @@ -1026,6 +889,7 @@ def _set_reloading_loading_files(self): self.mock_subprocess.side_effect = [self.mock_ls_process] def _set_loading_files(self): + responses.calls.reset() self.mock_subprocess.reset_mock() self.mock_subprocess.side_effect = [self.mock_ls_process] + [ mock_opened_file(i) for i in range(len(OPENED_RUN_JSON_FILES) - 1) @@ -1053,7 +917,7 @@ def _assert_expected_loading_file_calls(self, single_call): def _additional_loading_logs(self, data_type, version): return [(f'==> gsutil cat gs://seqr-hail-search-data/v3.1/{data_type.replace("SV", "GCNV")}/runs/{version}/metadata.json', None)] - def _assert_expected_airtable_calls(self, has_reload_calls, single_call): + def _assert_expected_airtable_calls(self, has_success_run, single_call): # Test request tracking updates for validation errors if single_call: fields = {'Status': 'Available in Seqr'} @@ -1065,8 +929,16 @@ def _assert_expected_airtable_calls(self, has_reload_calls, single_call): self.assertDictEqual(json.loads(update_loading_tracking_request.body), {'records': [ {'id': 'rec12345', 'fields': fields}, ]}) - if not has_reload_calls: - return 0, 2 + + num_calls = 2 + if single_call: + num_calls = 8 + elif has_success_run: + num_calls = 9 + self.assertEqual(len(responses.calls), num_calls) + + if not has_success_run: + return # Test airtable PDO updates update_pdos_request = responses.calls[1].request @@ -1098,15 +970,3 @@ def _assert_expected_airtable_calls(self, has_reload_calls, single_call): self.assertDictEqual(json.loads(update_samples_request_2.body), {'records': [ {'id': 'recfMYDEZpPtzAIeV', 'fields': {'PDOID': ['rec0ABC123']}}, ]}) - - if single_call: - return 8, 0 - return 7, 2 - - def _assert_saved_variant_models_updated(self): - saved_variant = SavedVariant.objects.get(key=100, family_id=14) - self.assertDictEqual(saved_variant.genotypes, {'I000018_na21234': { - 'ab': 0.0, 'dp': 49, 'gq': 99, 'numAlt': 2, 'filters': [], - 'sampleId': 'NA21234', 'familyGuid': 'F000014_14', 'sampleType': 'WGS', 'individualGuid': 'I000018_na21234', - }}) - self.assertDictEqual(saved_variant.saved_variant_json, {}) From 694df3b5421070f0b9fdbd4e1dcf7dae1f3edac4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 22 Aug 2025 18:16:12 -0400 Subject: [PATCH 17/89] remove vlm hail backend --- .github/workflows/vlm-unit-tests.yaml | 26 ++------------------------ vlm/hail_backend_utils.py | 15 --------------- vlm/match.py | 7 ++----- 3 files changed, 4 insertions(+), 44 deletions(-) delete mode 100644 vlm/hail_backend_utils.py diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 35c1c631e8..f8dcf4d23b 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -12,38 +12,16 @@ on: paths: - 'vlm/**' - '.github/workflows/*vlm*.yaml' - - 'hail_search/fixtures/*' pull_request: types: [opened, synchronize, reopened] paths: - 'vlm/**' - '.github/workflows/*vlm*.yaml' - - 'hail_search/fixtures/*' jobs: - vlm_hail_backend: - runs-on: ubuntu-latest - container: hailgenetics/hail:0.2.128 - - steps: - - uses: actions/checkout@v2 - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip wheel - pip install -r vlm/requirements-test.txt - - name: Run coverage tests - run: | - export VLM_DATA_DIR=./hail_search/fixtures - export SEQR_BASE_URL=https://test-seqr.org/ - export NODE_ID=TestVLM - export MACHINE_MEM=24 - export JAVA_OPTS_XSS=16M - coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py","./vlm/clickhouse_utils.py" -m pytest vlm/ - coverage report --fail-under=95 - vlm_clickhouse: runs-on: ubuntu-latest - container: hailgenetics/hail:0.2.128 + container: hailgenetics/hail:0.2.128 # TODO different image? services: clickhouse: @@ -78,6 +56,6 @@ jobs: export CLICKHOUSE_VLM_USERNAME=vlm_test_user export CLICKHOUSE_VLM_PASSWORD=vlm_test_password export CLICKHOUSE_DATABASE=test_seqr - coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py","./vlm/hail_backend_utils.py" -m pytest vlm/ + coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py" -m pytest vlm/ coverage report --fail-under=95 diff --git a/vlm/hail_backend_utils.py b/vlm/hail_backend_utils.py deleted file mode 100644 index 6d12287870..0000000000 --- a/vlm/hail_backend_utils.py +++ /dev/null @@ -1,15 +0,0 @@ -import hail as hl -import os - -VLM_DATA_DIR = os.environ.get('VLM_DATA_DIR') - - -def get_hail_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, genome_build: str) -> hl.Struct: - interval = hl.eval(hl.interval(locus, locus, includes_start=True, includes_end=True)) - ht = hl.read_table( - f'{VLM_DATA_DIR}/{genome_build}/SNV_INDEL/annotations.ht', _intervals=[interval], _filter_intervals=True, - ) - ht = ht.filter(ht.alleles == hl.array([ref, alt])) - - counts = ht.aggregate(hl.agg.take(ht.gt_stats, 1)) - return (counts[0].AC, counts[0].hom) if counts else (0, 0) diff --git a/vlm/match.py b/vlm/match.py index 0795e0f87b..8c2b42cd62 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -3,9 +3,7 @@ import os from vlm.clickhouse_utils import get_clickhouse_variant_counts -from vlm.hail_backend_utils import get_hail_variant_counts -CLICKHOUSE_ENABLED = bool(os.environ.get('CLICKHOUSE_SERVICE_HOSTNAME')) SEQR_BASE_URL = os.environ.get('SEQR_BASE_URL') VLM_DEFAULT_CONTACT_EMAIL = os.environ.get('VLM_DEFAULT_CONTACT_EMAIL') NODE_ID = os.environ.get('NODE_ID') @@ -42,12 +40,11 @@ def get_variant_match(query: dict) -> dict: chrom, pos, ref, alt, genome_build = _parse_match_query(query) locus = hl.locus(chrom, pos, reference_genome=genome_build) - get_counts_func = get_clickhouse_variant_counts if CLICKHOUSE_ENABLED else get_hail_variant_counts - ac, hom = get_counts_func(locus, ref, alt, genome_build) + ac, hom = get_clickhouse_variant_counts(locus, ref, alt, genome_build) liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 liftover_locus = hl.liftover(locus, liftover_genome_build) - lift_ac, lift_hom = get_counts_func(liftover_locus, ref, alt, liftover_genome_build) + lift_ac, lift_hom = get_clickhouse_variant_counts(liftover_locus, ref, alt, liftover_genome_build) url = _get_contact_url( chrom, pos, ref, alt, genome_build, liftover_genome_build, liftover_locus if lift_ac and not ac else None, From 20afb6aaf2bfb897060ee5b549a03af2f211f225 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 9 Sep 2025 16:23:14 -0400 Subject: [PATCH 18/89] clean up --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index f8dcf4d23b..c5a570784e 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -21,7 +21,7 @@ on: jobs: vlm_clickhouse: runs-on: ubuntu-latest - container: hailgenetics/hail:0.2.128 # TODO different image? + container: hailgenetics/hail:0.2.128 services: clickhouse: From b84a7aa73d21d2cc3672eae1cd4b10a70f2d8307 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 Aug 2025 12:48:29 -0400 Subject: [PATCH 19/89] clean up check samples job --- .../check_for_new_samples_from_pipeline.py | 126 +----------------- seqr/utils/search/utils.py | 4 +- seqr/views/utils/variant_utils.py | 3 - 3 files changed, 7 insertions(+), 126 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 40033f03c7..ad4045841e 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -8,14 +8,11 @@ import re from clickhouse_search.search import get_clickhouse_genotypes -from reference_data.models import GENOME_VERSION_LOOKUP, GENOME_VERSION_GRCh38 +from reference_data.models import GENOME_VERSION_LOOKUP from seqr.models import Family, Sample, SavedVariant, Project, Individual from seqr.utils.communication_utils import safe_post_to_slack, send_project_email from seqr.utils.file_utils import file_iter, list_files, is_google_bucket_file_path from seqr.utils.search.add_data_utils import notify_search_data_loaded, update_airtable_loading_tracking_status -from seqr.utils.search.utils import parse_valid_variant_id, backend_specific_call -from seqr.utils.search.hail_search_utils import hail_variant_multi_lookup, search_data_type -from seqr.utils.xpos_utils import get_xpos, CHROMOSOMES, MIN_POS, MAX_POS from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS from seqr.views.utils.dataset_utils import match_and_update_search_samples from seqr.views.utils.export_utils import write_multiple_files @@ -29,7 +26,6 @@ CLICKHOUSE_MIGRATION_SENTINEL = 'hail_search_to_clickhouse_migration' RUN_FILE_PATH_TEMPLATE = '{data_dir}/{genome_version}/{dataset_type}/runs/{run_version}/{file_name}' -SUCCESS_FILE_NAME = '_SUCCESS' CLICKHOUSE_SUCCESS_FILE_NAME = '_CLICKHOUSE_LOAD_SUCCESS' VALIDATION_ERRORS_FILE_NAME = 'validation_errors.json' ERRORS_REPORTED_FILE_NAME = '_ERRORS_REPORTED' @@ -69,7 +65,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): runs = self._get_runs(**options) - success_file_name = backend_specific_call(None, SUCCESS_FILE_NAME, CLICKHOUSE_SUCCESS_FILE_NAME) + success_file_name = CLICKHOUSE_SUCCESS_FILE_NAME success_run_dirs = [run_dir for run_dir, run_details in runs.items() if success_file_name in run_details['files']] if success_run_dirs: self._load_success_runs(runs, success_run_dirs) @@ -95,38 +91,19 @@ def _load_success_runs(self, runs, success_run_dirs): return logger.info(f'Loading new samples from {len(success_run_dirs)} run(s)') - updated_families_by_data_type = defaultdict(set) - updated_variants_by_data_type = defaultdict(dict) for run_dir, run_details in new_runs.items(): try: if CLICKHOUSE_MIGRATION_SENTINEL in run_details["run_version"]: logging.info(f'Skipping ClickHouse migration {run_details["genome_version"]}/{run_details["dataset_type"]}: {run_details["run_version"]}') continue metadata_path = os.path.join(run_dir, 'metadata.json') - data_type, updated_families, updated_variants_by_id = self._load_new_samples(metadata_path, **run_details) - data_type_key = (data_type, run_details['genome_version']) - updated_families_by_data_type[data_type_key].update(updated_families) - updated_variants_by_data_type[data_type_key].update(updated_variants_by_id) + self._load_new_samples(metadata_path, **run_details) except Exception as e: logger.error(f'Error loading {run_details["run_version"]}: {e}') # Reset cached results for all projects, as seqr AFs will have changed for all projects when new data is added reset_cached_search_results(project=None) - backend_specific_call( - lambda *args: None, self._reload_dataset_type_shared_variant_annotations, lambda *args: None, - )(updated_families_by_data_type, updated_variants_by_data_type) - - @classmethod - def _reload_dataset_type_shared_variant_annotations(cls, updated_families_by_data_type, updated_variants_by_data_type): - for data_type_key, updated_families in updated_families_by_data_type.items(): - try: - cls._reload_shared_variant_annotations( - *data_type_key, updated_variants_by_data_type[data_type_key], exclude_families=updated_families, - ) - except Exception as e: - logger.error(f'Error reloading shared annotations for {"/".join(data_type_key)}: {e}') - @classmethod def _get_runs(cls, **kwargs): path = cls._run_path(lambda field: kwargs.get(field, '*') or '*') @@ -279,12 +256,9 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers logger.error(f'Error updating individuals sample qc {run_version}: {e}') # Reload saved variant JSON - update_function = backend_specific_call(None, None, cls._update_project_saved_variant_genotypes) - updated_variants_by_id = update_projects_saved_variant_json([ + update_projects_saved_variant_json([ (project.id, project.guid, project.name, project.genome_version, families) for project, families in families_by_project.items() - ], user_email=USER_EMAIL, dataset_type=dataset_type, update_function=update_function, samples=updated_samples, clickhouse_dataset_type=clickhouse_dataset_type) - - return search_data_type(dataset_type, sample_type), set(family_project_map.keys()), updated_variants_by_id + ], user_email=USER_EMAIL, dataset_type=dataset_type, update_function=cls._update_project_saved_variant_genotypes, samples=updated_samples, clickhouse_dataset_type=clickhouse_dataset_type) @classmethod def _is_internal_project(cls, project): @@ -440,96 +414,6 @@ def _update_project_saved_variant_genotypes(cls, project_id, genome_version, use updates.update({v.id: v for v in variants}) return updates - @classmethod - def _reload_shared_variant_annotations(cls, data_type, genome_version, updated_variants_by_id=None, exclude_families=None, chromosomes=None): - dataset_type = data_type.split('_')[0] - is_sv = dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS) - dataset_type = data_type.split('_')[0] if is_sv else data_type - db_genome_version = genome_version.replace('GRCh', '') - updated_annotation_samples = Sample.objects.filter( - is_active=True, dataset_type=dataset_type, - individual__family__project__genome_version=db_genome_version, - ) - if exclude_families: - updated_annotation_samples = updated_annotation_samples.exclude(individual__family__guid__in=exclude_families) - if is_sv: - updated_annotation_samples = updated_annotation_samples.filter(sample_type=data_type.split('_')[1]) - - variant_models = get_saved_variants( - genome_version, dataset_type=dataset_type, - family_guids=updated_annotation_samples.values_list('individual__family__guid', flat=True).distinct(), - ) - - variant_type_summary = f'{data_type} {genome_version} saved variants' - - if updated_variants_by_id: - fetched_variant_models = variant_models.filter(variant_id__in=updated_variants_by_id.keys()) - if fetched_variant_models: - logger.info(f'Reloading shared annotations for {len(fetched_variant_models)} fetched {variant_type_summary}') - for variant_model in fetched_variant_models: - updated_variant = { - k: v for k, v in updated_variants_by_id[variant_model.variant_id].items() - if k not in {'familyGuids', 'genotypes'} - } - variant_model.saved_variant_json.update(updated_variant) - SavedVariant.bulk_update_models(None, fetched_variant_models, ['saved_variant_json']) - - variant_models = variant_models.exclude(variant_id__in=updated_variants_by_id.keys()) - - chromosomes = cls._get_chroms_to_reload(chromosomes, dataset_type, genome_version, variant_models) - if chromosomes is None: - return - - for chrom in chromosomes: - cls._reload_shared_variant_annotations_by_chrom(chrom, variant_models, data_type, genome_version, variant_type_summary) - - @staticmethod - def _get_chroms_to_reload(chromosomes, dataset_type, genome_version, variant_models): - if dataset_type != Sample.DATASET_TYPE_VARIANT_CALLS: - return [None] - if chromosomes: - return chromosomes - - num_reload = variant_models.count() - if genome_version == GENOME_VERSION_LOOKUP[GENOME_VERSION_GRCh38] and num_reload > MAX_RELOAD_VARIANTS: - logger.info(f'Skipped reloading all {num_reload} saved variant annotations for {dataset_type} {genome_version}') - return None - - return CHROMOSOMES - - @classmethod - def _reload_shared_variant_annotations_by_chrom(cls, chrom, variant_models, data_type, genome_version, variant_type_summary): - if chrom: - variant_models = variant_models.filter(xpos__gte=get_xpos(chrom, MIN_POS), xpos__lte=get_xpos(chrom, MAX_POS)) - - chrom_summary = f' in chromosome {chrom}' if chrom else '' - if not variant_models: - logger.info(f'No additional {variant_type_summary} to update{chrom_summary}') - return - - variants_by_id = defaultdict(list) - for v in variant_models: - variants_by_id[v.variant_id].append(v) - - logger.info(f'Reloading shared annotations for {len(variant_models)} {variant_type_summary}{chrom_summary} ({len(variants_by_id)} unique)') - - variant_ids = sorted(variants_by_id.keys()) - if chrom: - variant_ids = sorted([parse_valid_variant_id(variant_id) for variant_id in variant_ids]) - - for i in range(0, len(variant_ids), MAX_LOOKUP_VARIANTS): - updated_variants = hail_variant_multi_lookup(USER_EMAIL, variant_ids[i:i+MAX_LOOKUP_VARIANTS], data_type, genome_version) - logger.info(f'Fetched {len(updated_variants)} additional variants{chrom_summary}') - - updated_variant_models = [] - for variant in updated_variants: - for variant_model in variants_by_id[variant['variantId']]: - variant_model.saved_variant_json.update(variant) - updated_variant_models.append(variant_model) - - SavedVariant.bulk_update_models(None, updated_variant_models, ['saved_variant_json']) - -reload_shared_variant_annotations = Command._reload_shared_variant_annotations update_individuals_sample_qc = Command._update_individuals_sample_qc get_pipeline_runs = Command._get_runs diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 93cee91617..20307a38d1 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -464,12 +464,12 @@ def _parse_variant_items(search_json): def parse_variant_id(variant_id): try: - return parse_valid_variant_id(variant_id) + return _parse_valid_variant_id(variant_id) except (KeyError, ValueError): return None -def parse_valid_variant_id(variant_id): +def _parse_valid_variant_id(variant_id): chrom, pos, ref, alt = variant_id.split('-') chrom = format_chrom(chrom) pos = int(pos) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 5f71832f7a..9c24d50d8a 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -39,7 +39,6 @@ def update_projects_saved_variant_json(projects, user_email, update_function=Non success = {} skipped = {} error = {} - updated_variants_by_id = {} logger.info(f'Reloading saved variants in {len(projects)} projects') for project_id, project_guid, project_name, genome_version, family_guids in tqdm(projects, unit=' project'): try: @@ -51,7 +50,6 @@ def update_projects_saved_variant_json(projects, user_email, update_function=Non success[project_name] = len(updated_saved_variants) family_summary = f' in {len(family_guids)} families' if family_guids else '' logger.info(f'Updated {len(updated_saved_variants)} variants{family_summary} for project {project_name}') - updated_variants_by_id.update({v.variant_id: v.saved_variant_json for v in updated_saved_variants.values()}) except Exception as e: traceback_message = traceback.format_exc() logger.error(traceback_message) @@ -68,7 +66,6 @@ def update_projects_saved_variant_json(projects, user_email, update_function=Non logger.info(f'{len(error)} failed projects') for k, v in error.items(): logger.info(f' {k}: {v}') - return updated_variants_by_id def get_saved_variants(genome_version, project_id=None, family_guids=None, dataset_type=None, clickhouse_dataset_type=None): From 9ecf3b85e4c7f973fb137b4664615b344743def2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 25 Aug 2025 16:56:15 -0400 Subject: [PATCH 20/89] fix local check samples test --- .../check_for_new_samples_from_pipeline.py | 2 - ...eck_for_new_samples_from_pipeline_tests.py | 244 ++++-------------- 2 files changed, 52 insertions(+), 194 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index ad4045841e..40239b4546 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -37,8 +37,6 @@ Sample.DATASET_TYPE_SV_CALLS: f'{Sample.DATASET_TYPE_SV_CALLS}_WGS', } USER_EMAIL = 'manage_command' -MAX_LOOKUP_VARIANTS = 1000 -MAX_RELOAD_VARIANTS = 100000 RELATEDNESS_CHECK_NAME = 'relatedness_check' PDO_COPY_FIELDS = [ diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 0700851413..0e6ec747c5 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -6,14 +6,12 @@ import mock import responses -from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, AuthenticationTestCase +from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, AuthenticationTestCase, DifferentDbTransactionSupportMixin from seqr.models import Project, Family, Individual, Sample, SavedVariant SEQR_URL = 'https://seqr.broadinstitute.org/' PROJECT_GUID = 'R0003_test' EXTERNAL_PROJECT_GUID = 'R0004_non_analyst_project' -MOCK_HAIL_HOST = 'test-hail-host' -MOCK_HAIL_ORIGIN = f'http://{MOCK_HAIL_HOST}' GUID_ID = 54321 GCNV_GUID_ID = 12345 @@ -161,12 +159,12 @@ '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/manual__2025-01-13/_ERRORS_REPORTED', '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/manual__2025-01-13/validation_errors.json', '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/manual__2025-01-14/validation_errors.json', - '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/auto__2023-08-09/_SUCCESS', - '/seqr/seqr-hail-search-data/GRCh37/SNV_INDEL/runs/manual__2023-11-02/_SUCCESS', - '/seqr/seqr-hail-search-data/GRCh38/MITO/runs/auto__2024-08-12/_SUCCESS', - '/seqr/seqr-hail-search-data/GRCh38/GCNV/runs/auto__2024-09-14/_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/auto__2023-08-09/_CLICKHOUSE_LOAD_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh37/SNV_INDEL/runs/manual__2023-11-02/_CLICKHOUSE_LOAD_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh38/MITO/runs/auto__2024-08-12/_CLICKHOUSE_LOAD_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh38/GCNV/runs/auto__2024-09-14/_CLICKHOUSE_LOAD_SUCCESS', '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/manual__2025-01-24/validation_errors.json', - '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/hail_search_to_clickhouse_migration_WGS_R0877_neptune/_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/hail_search_to_clickhouse_migration_WGS_R0877_neptune/_CLICKHOUSE_LOAD_SUCCESS', ] RUN_PATHS = [ b'gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-13/', @@ -375,10 +373,9 @@ def set_up(self): self.addCleanup(patcher.stop) Sample.objects.filter(guid=OLD_DATA_SAMPLE_GUID).update(sample_type='WES') - def _test_call(self, error_logs=None, reload_annotations_logs=None, run_loading_logs=None, reload_calls=None, num_runs=5, has_reload_calls=False): + def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=5): self._set_loading_files() self.reset_logs() - responses.calls.reset() call_command('check_for_new_samples_from_pipeline') @@ -407,51 +404,23 @@ def _test_call(self, error_logs=None, reload_annotations_logs=None, run_loading_ {'severity': 'ERROR', '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent'}, )) logs.append(('Reset 2 cached results', None)) - logs += reload_annotations_logs or [] logs += [] if single_call else [(log, None) for log in self.VALIDATION_LOGS] logs.append(('DONE', None)) + self.maxDiff = None self.assert_json_logs(user=None, expected=logs) self.mock_redis.return_value.delete.assert_called_with('search_results__*', 'variant_lookup_results__*') - # Test reload saved variants - num_airtable_loading_calls, num_airtable_validation_calls = self._assert_expected_airtable_calls(has_reload_calls, single_call) - if not reload_calls: - self.assertEqual(len(responses.calls), num_airtable_loading_calls + num_airtable_validation_calls) - return - - reload_annotation_calls = [] if single_call else [ - {'genome_version': 'GRCh38', 'data_type': 'SNV_INDEL', 'variant_ids': [['1', 1562437, 'G', 'CA']]}, - {'genome_version': 'GRCh38', 'data_type': 'SNV_INDEL', 'variant_ids': [['1', 1562437, 'G', 'CA']]}, - {'genome_version': 'GRCh38', 'data_type': 'SNV_INDEL', 'variant_ids': [['1', 46859832, 'G', 'A']]}, - {'genome_version': 'GRCh38', 'data_type': 'SV_WES', 'variant_ids': ['prefix_19107_DEL']} - ] - self.assertEqual(len(responses.calls), len(reload_calls) +len(reload_annotation_calls) + num_airtable_loading_calls + num_airtable_validation_calls) - for i, call in enumerate(reload_calls or []): - resp = responses.calls[i+num_airtable_loading_calls] - self.assertEqual(resp.request.url, f'{MOCK_HAIL_ORIGIN}:5000/search') - self.assertEqual(resp.request.headers.get('From'), 'manage_command') - self.assertDictEqual(json.loads(resp.request.body), call) - - for i, body in enumerate(reload_annotation_calls): - multi_lookup_request = responses.calls[num_airtable_loading_calls+len(reload_calls)+i].request - self.assertEqual(multi_lookup_request.url, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup') - self.assertEqual(multi_lookup_request.headers.get('From'), 'manage_command') - self.assertDictEqual(json.loads(multi_lookup_request.body), body) + self._assert_expected_airtable_calls(bool(run_loading_logs), single_call) def _additional_loading_logs(self, data_type, version): return [] - @mock.patch('seqr.management.commands.check_for_new_samples_from_pipeline.MAX_LOOKUP_VARIANTS', 1) @mock.patch('seqr.views.utils.airtable_utils.BASE_URL', 'https://test-seqr.org/') @mock.patch('seqr.views.utils.airtable_utils.MAX_UPDATE_RECORDS', 2) - @mock.patch('seqr.management.commands.check_for_new_samples_from_pipeline.MAX_RELOAD_VARIANTS') @mock.patch('seqr.views.utils.export_utils.TemporaryDirectory') @mock.patch('seqr.utils.communication_utils.EmailMultiAlternatives') - @responses.activate - def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): - mock_max_reload_variants.__lt__.return_value = False - + def test_command(self, mock_email, mock_temp_dir): # Test errors self._set_empty_loading_files() with self.assertRaises(CommandError) as ce: @@ -485,69 +454,33 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): # Test success self.mock_send_slack.reset_mock() mock_email.reset_mock() - search_body = { - 'genome_version': 'GRCh38', 'num_results': 1, 'variant_ids': [['1', 248367227, 'TC', 'T']], 'variant_keys': [], - } - reload_snv_indel_calls = [ - {**search_body, 'sample_data': {'SNV_INDEL': [ - {'individual_guid': 'I000016_na20888', 'family_guid': 'F000012_12', 'project_guid': 'R0003_test', 'affected': 'A', 'sample_id': 'NA20888', 'sample_type': 'WES'}, - {'individual_guid': 'I000017_na20889', 'family_guid': 'F000012_12', 'project_guid': 'R0003_test', 'affected': 'A', 'sample_id': 'NA20889', 'sample_type': 'WES'}, - ]}}, - {**search_body, 'sample_data': {'SNV_INDEL': [ - {'individual_guid': 'I000018_na21234', 'family_guid': 'F000014_14', 'project_guid': 'R0004_non_analyst_project', 'affected': 'A', 'sample_id': 'NA21234', 'sample_type': 'WES'}, - ]}}, - ] - reload_fetched_annotations_logs = [ - ('Reloading shared annotations for 1 fetched SNV_INDEL GRCh38 saved variants', None), - ('update 1 SavedVariants', {'dbUpdate': { - 'dbEntity': 'SavedVariant', - 'entityIds': ['SV0000002_1248367227_r0390_100'], - 'updateFields': ['saved_variant_json'], - 'updateType': 'bulk_update'}, - }), - ] create_snv_indel_samples_logs = [ ('Loading 4 WES SNV_INDEL samples in 2 projects', None), ('create 4 Samples', {'dbUpdate': mock.ANY}), ('update 4 Samples', {'dbUpdate': mock.ANY}), ] - update_sample_qc_logs = [ + update_sample_logs = [ ('update 2 Individuals', {'dbUpdate': { 'dbEntity': 'Individual', 'entityIds': ['I000001_na19675', 'I000015_na20885'], 'updateFields': ['filter_flags', 'pop_platform_filters', 'population'], 'updateType': 'bulk_update'}} - ), + ), + ('Reloading saved variants in 2 projects', None), + ('Reloading genotypes for 0 SNV_INDEL variants in family F000012_12', None), + ('Updated 0 variants in 2 families for project Test Reprocessed Project', None), + ('Reloading genotypes for 1 SNV_INDEL variants in family F000014_14', None), + ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), + ('Updated 1 variants in 1 families for project Non-Analyst Project', None), + ('Reload Summary: ', None), + (' Non-Analyst Project: Updated 1 variants', None), ] - self._test_call(reload_calls=reload_snv_indel_calls + [ - {'genome_version': 'GRCh38', 'num_results': 1, 'variant_ids': [], 'variant_keys': ['prefix_19107_DEL'], 'sample_data': {'SV_WES': [ - {'individual_guid': 'I000017_na20889', 'family_guid': 'F000012_12', 'project_guid': 'R0003_test', 'affected': 'A', 'sample_id': 'NA20889', 'sample_type': 'WES'}, - ]}}, - ], reload_annotations_logs=reload_fetched_annotations_logs + [ - ('Reloading shared annotations for 2 SNV_INDEL GRCh38 saved variants in chromosome 1 (2 unique)', None), - ('Fetched 1 additional variants in chromosome 1', None), - ('update 1 SavedVariants', {'dbUpdate': { - 'dbEntity': 'SavedVariant', - 'entityIds': ['SV0059956_11560662_f019313_1'], - 'updateFields': ['saved_variant_json'], - 'updateType': 'bulk_update'}, - }), - ('Fetched 1 additional variants in chromosome 1', None), - ('update 1 SavedVariants', {'dbUpdate': { - 'dbEntity': 'SavedVariant', - 'entityIds': ['SV0059956_11560662_f019313_1'], - 'updateFields': ['saved_variant_json'], - 'updateType': 'bulk_update'}, - }), - ] + [(f'No additional SNV_INDEL GRCh38 saved variants to update in chromosome {chrom}', None) for chrom in ['2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', 'X', 'Y', 'M']] + [ - ('Reloading shared annotations for 1 SV_WES GRCh38 saved variants (1 unique)', None), - ('Fetched 1 additional variants', None), - ], run_loading_logs={ + self._test_call(run_loading_logs={ 'GRCh38/SNV_INDEL': create_snv_indel_samples_logs + [ ('update 1 Samples', {'dbUpdate': mock.ANY}), ('update 2 Familys', {'dbUpdate': mock.ANY}), ] + self.AIRTABLE_LOGS + [ ('update 3 Familys', {'dbUpdate': mock.ANY}), - ] + update_sample_qc_logs + self.RELOAD_PROJECT_VARIANT_LOGS, + ] + update_sample_logs, 'GRCh38/MITO': [ ('Loading 2 WGS MITO samples in 1 projects', None) ], @@ -558,7 +491,10 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): ('update 3 Samples', {'dbUpdate': mock.ANY}), ('update 1 Familys', {'dbUpdate': mock.ANY}), ('Reloading saved variants in 2 projects', None), - ] + self.RELOAD_SV_VARIANT_LOGS, + ('Updated 0 variants in 1 families for project 1kg project nåme with uniçøde', None), + ('Updated 0 variants in 1 families for project Test Reprocessed Project', None), + ('Reload Summary: ', None), + ], }, error_logs={ 'manual__2023-11-02': 'Invalid families in run metadata GRCh37/SNV_INDEL: manual__2023-11-02 - F0000123_ABC', 'auto__2024-08-12': 'Matches not found for sample ids: NA20885, NA22882', @@ -650,7 +586,11 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): ) self.assertEqual(Family.objects.get(guid='F000014_14').analysis_status, 'Rncc') - self._assert_saved_variant_models_updated() + saved_variant = SavedVariant.objects.get(key=100, family_id=14) + self.assertDictEqual(saved_variant.genotypes, {'I000018_na21234': { + 'ab': 0.0, 'dp': 49, 'gq': 99, 'numAlt': 2, 'filters': [], + 'sampleId': 'NA21234', 'familyGuid': 'F000014_14', 'sampleType': 'WGS', 'individualGuid': 'I000018_na21234', + }}) # Test notifications self.assertEqual(self.mock_send_slack.call_count, 7 + len(self.ADDITIONAL_SLACK_CALLS)) @@ -741,20 +681,18 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): self.mock_redis.return_value.delete.assert_not_called() # Test reloading shared annotations is skipped if too many saved variants - mock_max_reload_variants.__lt__.return_value = True snv_indel_samples.delete() airtable_logs = self.AIRTABLE_LOGS[:-1] if self.AIRTABLE_LOGS: airtable_logs.append(('Fetched 1 AnVIL Seqr Loading Requests Tracking records from airtable', None)) - self._test_call(num_runs=2, reload_annotations_logs=reload_fetched_annotations_logs + [ - ('Skipped reloading all 2 saved variant annotations for SNV_INDEL GRCh38', None), - ], reload_calls=reload_snv_indel_calls, run_loading_logs={ - 'GRCh38/SNV_INDEL': create_snv_indel_samples_logs + airtable_logs + update_sample_qc_logs + self.RELOAD_PROJECT_VARIANT_LOGS, + self._test_call(num_runs=2, run_loading_logs={ + 'GRCh38/SNV_INDEL': create_snv_indel_samples_logs + airtable_logs + update_sample_logs, }) -@mock.patch('seqr.utils.search.hail_search_utils.HAIL_BACKEND_SERVICE_HOSTNAME', MOCK_HAIL_HOST) -class LocalCheckNewSamplesTest(AuthenticationTestCase, CheckNewSamplesTest): - fixtures = ['users', '1kg_project'] + +class LocalCheckNewSamplesTest(DifferentDbTransactionSupportMixin, AuthenticationTestCase, CheckNewSamplesTest): + fixtures = ['users', '1kg_project', 'clickhouse_saved_variants'] + databases = '__all__' ES_HOSTNAME = '' @@ -766,22 +704,6 @@ class LocalCheckNewSamplesTest(AuthenticationTestCase, CheckNewSamplesTest): LIST_FILE_LOGS = [] AIRTABLE_LOGS = [] VALIDATION_LOGS = [] - RELOAD_PROJECT_VARIANT_LOGS = [ - ('Reloading saved variants in 2 projects', None), - ('Updated 0 variants in 2 families for project Test Reprocessed Project', None), - ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), - ('Updated 1 variants in 1 families for project Non-Analyst Project', None), - ('Reload Summary: ', None), - (' Non-Analyst Project: Updated 1 variants', None), - ] - RELOAD_SV_VARIANT_LOGS = [ - (mock.ANY, {'severity': 'ERROR', '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent'}), - ('Error reloading variants in Test Reprocessed Project: Bad Request', {'severity': 'ERROR','@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent'}), - ('Reload Summary: ', None), - ('Skipped the following 1 project with no saved variants: 1kg project nåme with uniçøde', None), - ('1 failed projects', None), - (' Test Reprocessed Project: Bad Request', None), - ] ADDITIONAL_SLACK_CALLS = [ mock.call( 'seqr-data-loading', @@ -809,26 +731,6 @@ def setUp(self): self.set_up() super().setUp() - def test_command(self, *args, **kwargs): - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=200, json={ - 'results': [{'variantId': '1-248367227-TC-T', 'familyGuids': ['F000014_14'], 'updated_field': 'updated_value'}], - 'total': 1, - }) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup', status=400) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup', status=200, json={ - 'results': [{'variantId': '1-46859832-G-A', 'updated_new_field': 'updated_value', 'rsid': 'rs123'}], - }) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=200, json={ - 'results': [{'variantId': '1-248367227-TC-T', 'familyGuids': ['F000014_14'], 'updated_field': 'updated_value'}], - 'total': 1, - }) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=400) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=200, json={ - 'results': [{'variantId': '1-248367227-TC-T', 'familyGuids': ['F000014_14'], 'updated_field': 'updated_value'}], - 'total': 1, - }) - super().test_command(*args, **kwargs) - def _set_empty_loading_files(self): self.mock_glob.return_value = [] @@ -850,7 +752,7 @@ def _assert_expected_loading_file_calls(self, single_call): self.mock_glob.assert_called_with('/seqr/seqr-hail-search-data/*/*/runs/*/*', recursive=False) self.mock_open.assert_has_calls([ mock.call(LOCAL_RUN_PATHS[2], 'r'), - *[mock.call(path.replace('_SUCCESS', 'metadata.json'), 'r') for path in LOCAL_RUN_PATHS[3:-1]] + *[mock.call(path.replace('_CLICKHOUSE_LOAD_SUCCESS', 'metadata.json'), 'r') for path in LOCAL_RUN_PATHS[3:-1]] ], any_order=True) self.assertEqual(self.mock_mkdir.call_count, 0 if single_call else 2) self.assertEqual(list(self.mock_written_files.keys()), [ @@ -859,28 +761,7 @@ def _assert_expected_loading_file_calls(self, single_call): ]) def _assert_expected_airtable_calls(self, *args, **kwargs): - return 0, 0 - - def _assert_saved_variant_models_updated(self): - updated_variants = SavedVariant.objects.filter(saved_variant_json__updated_field='updated_value') - self.assertEqual(len(updated_variants), 2) - self.assertSetEqual( - {v.guid for v in updated_variants}, - {'SV0000006_1248367227_r0004_non', 'SV0000002_1248367227_r0390_100'} - ) - reloaded_variant = next(v for v in updated_variants if v.guid == 'SV0000006_1248367227_r0004_non') - annotation_updated_variant = next(v for v in updated_variants if v.guid == 'SV0000002_1248367227_r0390_100') - self.assertEqual(len(reloaded_variant.saved_variant_json), 3) - self.assertListEqual(reloaded_variant.saved_variant_json['familyGuids'], ['F000014_14']) - self.assertEqual(len(annotation_updated_variant.saved_variant_json), 17) - self.assertListEqual(annotation_updated_variant.saved_variant_json['familyGuids'], ['F000001_1']) - - annotation_updated_json = SavedVariant.objects.get(guid='SV0059956_11560662_f019313_1').saved_variant_json - self.assertEqual(len(annotation_updated_json), 16) - self.assertEqual(annotation_updated_json['updated_new_field'], 'updated_value') - self.assertEqual(annotation_updated_json['rsid'], 'rs123') - self.assertEqual(annotation_updated_json['mainTranscriptId'], 'ENST00000505820') - self.assertEqual(len(annotation_updated_json['genotypes']), 3) + pass class AirtableCheckNewSamplesTest(AnvilAuthenticationTestCase, CheckNewSamplesTest): @@ -925,21 +806,6 @@ class AirtableCheckNewSamplesTest(AnvilAuthenticationTestCase, CheckNewSamplesTe '==> gsutil cat gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-24/validation_errors.json', '==> gsutil mv /mock/tmp/* gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-24/', ] - RELOAD_PROJECT_VARIANT_LOGS = [ - ('Reloading saved variants in 2 projects', None), - ('Reloading genotypes for 0 SNV_INDEL variants in family F000012_12', None), - ('Updated 0 variants in 2 families for project Test Reprocessed Project', None), - ('Reloading genotypes for 1 SNV_INDEL variants in family F000014_14', None), - ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), - ('Updated 1 variants in 1 families for project Non-Analyst Project', None), - ('Reload Summary: ', None), - (' Non-Analyst Project: Updated 1 variants', None), - ] - RELOAD_SV_VARIANT_LOGS = [ - ('Updated 0 variants in 1 families for project 1kg project nåme with uniçøde', None), - ('Updated 0 variants in 1 families for project Test Reprocessed Project', None), - ('Reload Summary: ', None), - ] ADDITIONAL_SLACK_CALLS = [ mock.call( 'anvil-data-loading', @@ -977,6 +843,7 @@ def setUp(self): self.set_up() super().setUp() + @responses.activate def test_command(self, *args, **kwargs): responses.add( responses.GET, @@ -1009,10 +876,6 @@ def test_command(self, *args, **kwargs): }))) super().test_command(*args, **kwargs) - def _test_call(self, *args, reload_annotations_logs=None, reload_calls=None, **kwargs): - # No reloading for clickhouse - super()._test_call(*args, has_reload_calls=bool(reload_calls), **kwargs) - def _set_empty_loading_files(self): self.mock_subprocess.return_value.communicate.return_value = b'', b'One or more URLs matched no objects' @@ -1026,6 +889,7 @@ def _set_reloading_loading_files(self): self.mock_subprocess.side_effect = [self.mock_ls_process] def _set_loading_files(self): + responses.calls.reset() self.mock_subprocess.reset_mock() self.mock_subprocess.side_effect = [self.mock_ls_process] + [ mock_opened_file(i) for i in range(len(OPENED_RUN_JSON_FILES) - 1) @@ -1053,7 +917,7 @@ def _assert_expected_loading_file_calls(self, single_call): def _additional_loading_logs(self, data_type, version): return [(f'==> gsutil cat gs://seqr-hail-search-data/v3.1/{data_type.replace("SV", "GCNV")}/runs/{version}/metadata.json', None)] - def _assert_expected_airtable_calls(self, has_reload_calls, single_call): + def _assert_expected_airtable_calls(self, has_success_run, single_call): # Test request tracking updates for validation errors if single_call: fields = {'Status': 'Available in Seqr'} @@ -1065,8 +929,16 @@ def _assert_expected_airtable_calls(self, has_reload_calls, single_call): self.assertDictEqual(json.loads(update_loading_tracking_request.body), {'records': [ {'id': 'rec12345', 'fields': fields}, ]}) - if not has_reload_calls: - return 0, 2 + + num_calls = 2 + if single_call: + num_calls = 8 + elif has_success_run: + num_calls = 9 + self.assertEqual(len(responses.calls), num_calls) + + if not has_success_run: + return # Test airtable PDO updates update_pdos_request = responses.calls[1].request @@ -1098,15 +970,3 @@ def _assert_expected_airtable_calls(self, has_reload_calls, single_call): self.assertDictEqual(json.loads(update_samples_request_2.body), {'records': [ {'id': 'recfMYDEZpPtzAIeV', 'fields': {'PDOID': ['rec0ABC123']}}, ]}) - - if single_call: - return 8, 0 - return 7, 2 - - def _assert_saved_variant_models_updated(self): - saved_variant = SavedVariant.objects.get(key=100, family_id=14) - self.assertDictEqual(saved_variant.genotypes, {'I000018_na21234': { - 'ab': 0.0, 'dp': 49, 'gq': 99, 'numAlt': 2, 'filters': [], - 'sampleId': 'NA21234', 'familyGuid': 'F000014_14', 'sampleType': 'WGS', 'individualGuid': 'I000018_na21234', - }}) - self.assertDictEqual(saved_variant.saved_variant_json, {}) From 8bcb0ed82c0e0ed436d1a799ef969def8e86e874 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Sep 2025 10:21:27 -0400 Subject: [PATCH 21/89] remove unsuable reload command --- .../reload_saved_variant_annotations.py | 27 ---- .../reload_saved_variant_annotations_tests.py | 131 ------------------ 2 files changed, 158 deletions(-) delete mode 100644 seqr/management/commands/reload_saved_variant_annotations.py delete mode 100644 seqr/management/tests/reload_saved_variant_annotations_tests.py diff --git a/seqr/management/commands/reload_saved_variant_annotations.py b/seqr/management/commands/reload_saved_variant_annotations.py deleted file mode 100644 index a2481e1731..0000000000 --- a/seqr/management/commands/reload_saved_variant_annotations.py +++ /dev/null @@ -1,27 +0,0 @@ -from django.core.management.base import BaseCommand, CommandError -from reference_data.models import GENOME_VERSION_LOOKUP -from seqr.models import Sample -from seqr.management.commands.check_for_new_samples_from_pipeline import reload_shared_variant_annotations -from seqr.utils.search.hail_search_utils import search_data_type -from seqr.utils.search.utils import backend_specific_call - -DATA_TYPE_CHOICES = { - search_data_type(dt, st) for dt in Sample.DATASET_TYPE_LOOKUP for st in [Sample.SAMPLE_TYPE_WGS, Sample.SAMPLE_TYPE_WES] -} - - -def _clickhouse_error(): - raise CommandError('Reloading variant annotations is not supported in clickhouse') - - -class Command(BaseCommand): - help = 'Reload shared variant annotations for all saved variants' - - def add_arguments(self, parser): - parser.add_argument('data_type', choices=sorted(DATA_TYPE_CHOICES)) - parser.add_argument('genome_version', choices=sorted(GENOME_VERSION_LOOKUP.values())) - parser.add_argument('chromosomes', nargs='*', help='Chromosome(s) to reload. If not specified, defaults to all chromosomes.') - - def handle(self, *args, **options): - backend_specific_call(lambda: True, lambda: True, _clickhouse_error)() - reload_shared_variant_annotations(options['data_type'], options['genome_version'], chromosomes=options['chromosomes']) diff --git a/seqr/management/tests/reload_saved_variant_annotations_tests.py b/seqr/management/tests/reload_saved_variant_annotations_tests.py deleted file mode 100644 index 9df44a79d6..0000000000 --- a/seqr/management/tests/reload_saved_variant_annotations_tests.py +++ /dev/null @@ -1,131 +0,0 @@ -from django.core.management import call_command -from django.core.management.base import CommandError -import json -import mock -import responses - -from seqr.views.utils.test_utils import AnvilAuthenticationTestCase -from seqr.models import Sample, SavedVariant - -MOCK_HAIL_HOST = 'test-hail-host' -MOCK_HAIL_ORIGIN = f'http://{MOCK_HAIL_HOST}' - - -@mock.patch('seqr.utils.search.hail_search_utils.HAIL_BACKEND_SERVICE_HOSTNAME', MOCK_HAIL_HOST) -class ReloadVariantAnnotationsTest(AnvilAuthenticationTestCase): - fixtures = ['users', '1kg_project'] - - CLICKHOUSE_HOSTNAME = None - - @responses.activate - def test_command(self): - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup', status=200, json={ - 'results': [ - {'variantId': '1-46859832-G-A', 'updated_new_field': 'updated_value', 'rsid': 'rs123'}, - {'variantId': '1-248367227-TC-T', 'updated_field': 'updated_value'}, - ], - }) - - # Test errors - with self.assertRaises(CommandError) as ce: - call_command('reload_saved_variant_annotations') - self.assertEqual(str(ce.exception), 'Error: the following arguments are required: data_type, genome_version, chromosomes') - - with self.assertRaises(CommandError) as ce: - call_command('reload_saved_variant_annotations', 'SV', 'GRCh37') - self.assertEqual(str(ce.exception), "Error: argument data_type: invalid choice: 'SV' (choose from 'MITO', 'SNV_INDEL', 'SV_WES', 'SV_WGS')") - - # Test success - self.reset_logs() - call_command('reload_saved_variant_annotations', 'SNV_INDEL', 'GRCh37') - - self.assert_json_logs(user=None, expected=[ - ('Reloading shared annotations for 2 SNV_INDEL GRCh37 saved variants in chromosome 1 (2 unique)', None), - ('Fetched 2 additional variants in chromosome 1', None), - ('update 2 SavedVariants', {'dbUpdate': { - 'dbEntity': 'SavedVariant', - 'entityIds': ['SV0000002_1248367227_r0390_100', 'SV0059956_11560662_f019313_1'], - 'updateFields': ['saved_variant_json'], - 'updateType': 'bulk_update'}, - }), - ] + [(f'No additional SNV_INDEL GRCh37 saved variants to update in chromosome {chrom}', None) for chrom in ['2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20']] + [ - ('Reloading shared annotations for 1 SNV_INDEL GRCh37 saved variants in chromosome 21 (1 unique)', None), - ('Fetched 2 additional variants in chromosome 21', None), - ] + [(f'No additional SNV_INDEL GRCh37 saved variants to update in chromosome {chrom}', None) for chrom in ['22', 'X', 'Y', 'M']]) - - self.assertEqual(len(responses.calls), 2) - for call in responses.calls: - self.assertEqual(call.request.url, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup') - self.assertEqual(call.request.headers.get('From'), 'manage_command') - self.assertDictEqual(json.loads(responses.calls[0].request.body), { - 'genome_version': 'GRCh37', - 'data_type': 'SNV_INDEL', - 'variant_ids': [['1', 46859832, 'G', 'A'], ['1', 248367227, 'TC', 'T']], - }) - self.assertDictEqual(json.loads(responses.calls[1].request.body), { - 'genome_version': 'GRCh37', - 'data_type': 'SNV_INDEL', - 'variant_ids': [['21', 3343353, 'GAGA', 'G']], - }) - - annotation_updated_json_1 = SavedVariant.objects.get(guid='SV0000002_1248367227_r0390_100').saved_variant_json - self.assertEqual(len(annotation_updated_json_1), 17) - self.assertListEqual(annotation_updated_json_1['familyGuids'], ['F000001_1']) - self.assertEqual(annotation_updated_json_1['updated_field'], 'updated_value') - - annotation_updated_json_2 = SavedVariant.objects.get(guid='SV0059956_11560662_f019313_1').saved_variant_json - self.assertEqual(len(annotation_updated_json_2), 16) - self.assertEqual(annotation_updated_json_2['updated_new_field'], 'updated_value') - self.assertEqual(annotation_updated_json_2['rsid'], 'rs123') - self.assertEqual(annotation_updated_json_2['mainTranscriptId'], 'ENST00000505820') - self.assertEqual(len(annotation_updated_json_2['genotypes']), 3) - - # Test chromosome subset - responses.calls.reset() - self.reset_logs() - call_command('reload_saved_variant_annotations', 'SNV_INDEL', 'GRCh37', '3', '21') - - self.assert_json_logs(user=None, expected=[(log, None) for log in [ - 'No additional SNV_INDEL GRCh37 saved variants to update in chromosome 3', - 'Reloading shared annotations for 1 SNV_INDEL GRCh37 saved variants in chromosome 21 (1 unique)', - 'Fetched 2 additional variants in chromosome 21', - ]]) - - self.assertEqual(len(responses.calls), 1) - self.assertDictEqual(json.loads(responses.calls[0].request.body), { - 'genome_version': 'GRCh37', - 'data_type': 'SNV_INDEL', - 'variant_ids': [['21', 3343353, 'GAGA', 'G']], - }) - - responses.calls.reset() - self.reset_logs() - call_command('reload_saved_variant_annotations', 'SNV_INDEL', 'GRCh37', '3', '6') - self.assert_json_logs(user=None, expected=[ - (f'No additional SNV_INDEL GRCh37 saved variants to update in chromosome {chrom}', None) - for chrom in [3, 6] - ]) - self.assertEqual(len(responses.calls), 0) - - # Test SVs - responses.calls.reset() - Sample.objects.filter(guid='S000147_na21234').update(individual_id=20) - call_command('reload_saved_variant_annotations', 'SV_WGS', 'GRCh37') - - self.assertEqual(len(responses.calls), 1) - self.assertDictEqual(json.loads(responses.calls[0].request.body), { - 'genome_version': 'GRCh37', - 'data_type': 'SV_WGS', - 'variant_ids': ['prefix_19107_DEL'], - }) - - -class ClickhouseReloadVariantAnnotationsTest(AnvilAuthenticationTestCase): - - fixtures = ['users', '1kg_project'] - - def test_command(self): - with self.assertRaises(CommandError) as ce: - call_command('reload_saved_variant_annotations', 'SNV_INDEL', 'GRCh37') - self.assertEqual(str(ce.exception), 'Reloading variant annotations is not supported in clickhouse') - From b11efde9cdb29abf63177343a7a20950b81b9f2a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Sep 2025 10:37:36 -0400 Subject: [PATCH 22/89] clean up --- .../check_for_new_samples_from_pipeline.py | 3 +-- ...check_for_new_samples_from_pipeline_tests.py | 17 ++++++----------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 40239b4546..27d38ed371 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -63,8 +63,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): runs = self._get_runs(**options) - success_file_name = CLICKHOUSE_SUCCESS_FILE_NAME - success_run_dirs = [run_dir for run_dir, run_details in runs.items() if success_file_name in run_details['files']] + success_run_dirs = [run_dir for run_dir, run_details in runs.items() if CLICKHOUSE_SUCCESS_FILE_NAME in run_details['files']] if success_run_dirs: self._load_success_runs(runs, success_run_dirs) if not success_run_dirs: diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 0e6ec747c5..c3ba7986c9 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -406,12 +406,12 @@ def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=5): logs.append(('Reset 2 cached results', None)) logs += [] if single_call else [(log, None) for log in self.VALIDATION_LOGS] logs.append(('DONE', None)) - self.maxDiff = None self.assert_json_logs(user=None, expected=logs) self.mock_redis.return_value.delete.assert_called_with('search_results__*', 'variant_lookup_results__*') - self._assert_expected_airtable_calls(bool(run_loading_logs), single_call) + num_calls = self._assert_expected_airtable_calls(bool(run_loading_logs), single_call) + self.assertEqual(len(responses.calls), num_calls) def _additional_loading_logs(self, data_type, version): return [] @@ -761,7 +761,7 @@ def _assert_expected_loading_file_calls(self, single_call): ]) def _assert_expected_airtable_calls(self, *args, **kwargs): - pass + return 0 class AirtableCheckNewSamplesTest(AnvilAuthenticationTestCase, CheckNewSamplesTest): @@ -930,15 +930,8 @@ def _assert_expected_airtable_calls(self, has_success_run, single_call): {'id': 'rec12345', 'fields': fields}, ]}) - num_calls = 2 - if single_call: - num_calls = 8 - elif has_success_run: - num_calls = 9 - self.assertEqual(len(responses.calls), num_calls) - if not has_success_run: - return + return 8 if single_call else 2 # Test airtable PDO updates update_pdos_request = responses.calls[1].request @@ -970,3 +963,5 @@ def _assert_expected_airtable_calls(self, has_success_run, single_call): self.assertDictEqual(json.loads(update_samples_request_2.body), {'records': [ {'id': 'recfMYDEZpPtzAIeV', 'fields': {'PDOID': ['rec0ABC123']}}, ]}) + + return 8 if single_call else 9 From 693c0772e982f7ca9318e5d811dd3bdc1da12ec1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Sep 2025 11:04:02 -0400 Subject: [PATCH 23/89] update manage tests --- .../tests/deactivate_project_search_tests.py | 7 ------- .../tests/reload_saved_variant_json_tests.py | 2 +- .../transfer_families_to_different_project_tests.py | 11 ----------- 3 files changed, 1 insertion(+), 19 deletions(-) diff --git a/seqr/management/tests/deactivate_project_search_tests.py b/seqr/management/tests/deactivate_project_search_tests.py index cbd13ae46b..2a35dc9a4c 100644 --- a/seqr/management/tests/deactivate_project_search_tests.py +++ b/seqr/management/tests/deactivate_project_search_tests.py @@ -98,10 +98,3 @@ def _assert_expected_delete(self): 7: (0, 0, 0, 0), 8: (0, 0, 0, 0), }) - - -class HailBackendDeactivateProjectSearchTest(AuthenticationTestCase, DeactivateProjectSearchTest): - fixtures = ['users', '1kg_project'] - - CLICKHOUSE_HOSTNAME = '' - ES_HOSTNAME = '' diff --git a/seqr/management/tests/reload_saved_variant_json_tests.py b/seqr/management/tests/reload_saved_variant_json_tests.py index 46d47bde1a..1c885dbd70 100644 --- a/seqr/management/tests/reload_saved_variant_json_tests.py +++ b/seqr/management/tests/reload_saved_variant_json_tests.py @@ -10,7 +10,7 @@ FAMILY_GUID = 'F000001_1' -@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') +@mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') class ReloadSavedVariantJsonTest(TestCase): fixtures = ['users', '1kg_project'] diff --git a/seqr/management/tests/transfer_families_to_different_project_tests.py b/seqr/management/tests/transfer_families_to_different_project_tests.py index 5e18dbf721..f696bbc7be 100644 --- a/seqr/management/tests/transfer_families_to_different_project_tests.py +++ b/seqr/management/tests/transfer_families_to_different_project_tests.py @@ -64,14 +64,3 @@ class TransferFamiliesClickhouseTest(TransferFamiliesTest, AnvilAuthenticationTe ('Clickhouse does not support deleting individual families from project. Manually delete SNV_INDEL data for F000002_2 in project R0001_1kg', None), ('Clickhouse does not support deleting individual families from project. Manually delete GCNV data for F000002_2 in project R0001_1kg', None), ] - - -class TransferFamiliesHailBackendTest(TransferFamiliesTest, AuthenticationTestCase): - fixtures = ['users', '1kg_project'] - - ES_HOSTNAME = '' - CLICKHOUSE_HOSTNAME = '' - - LOGS = [ - ('Disabled search for 7 samples in the following 1 families: 2', None), - ] From 866b2d2d214e5f3a8c6ba08b0748d3112316a09c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Sep 2025 11:14:54 -0400 Subject: [PATCH 24/89] fix trigger dag --- seqr/views/apis/data_manager_api.py | 2 +- seqr/views/apis/data_manager_api_tests.py | 15 --------------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 5910d97003..b7bd5e109e 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -451,7 +451,7 @@ def _trigger_data_update(clickhouse_func, request_json, project, *args): info = [] for sample_type in sample_types: info.append(backend_specific_call( - _raise_backend_not_implemented, _raise_backend_not_implemented, clickhouse_func, + _raise_backend_not_implemented, clickhouse_func, )(project, *args, dataset_type=dataset_type, sample_type=sample_type)) return create_json_response({'info': info}) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 4cb5a6f4b1..cd5373f203 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -2081,18 +2081,3 @@ def _assert_expected_airtable_errors(self, url): self.assertDictEqual(response.json(), { 'error': 'The following samples are associated with misconfigured PDOs in Airtable: HG00731, NA21234', }) - -@mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') -class HailBackendDataManagerAPITest(LocalDataManagerAPITest): - fixtures = ['users', 'social_auth', '1kg_project', 'reference_data'] - - CLICKHOUSE_HOSTNAME = '' - ES_HOSTNAME = '' - - def _assert_expected_es_status(self, response): - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['error'], 'Elasticsearch is disabled') - - def _assert_expected_delete_index_response(self, response): - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['error'], 'Deleting indices is disabled without the elasticsearch backend') From d5c099f89c5494e3526c671bdfc0bdc8a72ed48a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Sep 2025 13:30:28 -0400 Subject: [PATCH 25/89] helper function for vcf iter --- seqr/utils/vcf_utils.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/seqr/utils/vcf_utils.py b/seqr/utils/vcf_utils.py index d9f49814a3..f51694119b 100644 --- a/seqr/utils/vcf_utils.py +++ b/seqr/utils/vcf_utils.py @@ -82,25 +82,15 @@ def validate_vcf_and_get_samples(data_path, user, genome_version, path_name=None return None byte_range = None if vcf_filename.endswith('.vcf') else (0, BLOCK_SIZE) - samples = {} - header = [] meta = defaultdict(dict) - for line in file_iter(vcf_filename, byte_range=byte_range): - line_str = line.decode() if isinstance(line, bytes) else line - if line_str.startswith('#'): - if line_str.startswith('#CHROM'): - header_cols = line_str.rstrip().split('\t') - format_indices = [index for index, col in enumerate(header_cols) if col == 'FORMAT'] - format_index = format_indices[0] + 1 if format_indices else len(header_cols) - header = header_cols[0:format_index] - samples = set(header_cols[format_index:]) - break - else: - meta_info = _get_vcf_meta_info(line_str) - if meta_info: - meta[meta_info['field']].update({meta_info['id']: meta_info['type']}) - else: - raise ErrorsWarningsException(['No header found in the VCF file.'], []) + header_line = next((_get_vcf_header_line(file_iter(vcf_filename, byte_range=byte_range), meta)), None) + if not header_line: + raise ErrorsWarningsException(['No header found in the VCF file.'], []) + header_cols = header_line.rstrip().split('\t') + format_indices = [index for index, col in enumerate(header_cols) if col == 'FORMAT'] + format_index = format_indices[0] + 1 if format_indices else len(header_cols) + header = header_cols[0:format_index] + samples = set(header_cols[format_index:]) _validate_vcf_header(header) if not samples: @@ -110,6 +100,18 @@ def validate_vcf_and_get_samples(data_path, user, genome_version, path_name=None return samples +def _get_vcf_header_line(vcf_file, meta): + for line in vcf_file: + line_str = line.decode() if isinstance(line, bytes) else line + if line_str.startswith('#'): + if line_str.startswith('#CHROM'): + yield line_str + else: + meta_info = _get_vcf_meta_info(line_str) + if meta_info: + meta[meta_info['field']].update({meta_info['id']: meta_info['type']}) + + def _validate_vcf_exists(data_path, user, path_name, allowed_exts): file_extensions = (allowed_exts or ()) + VCF_FILE_EXTENSIONS if not data_path.endswith(file_extensions): From 1e38a3c5e7facb8c30d10b46f1869e23db265c42 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Sep 2025 13:49:04 -0400 Subject: [PATCH 26/89] handle invalid utf8 encoding in vcf --- seqr/utils/vcf_utils.py | 9 +++++++-- seqr/views/apis/anvil_workspace_api_tests.py | 8 ++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/seqr/utils/vcf_utils.py b/seqr/utils/vcf_utils.py index f51694119b..bf38cde5d1 100644 --- a/seqr/utils/vcf_utils.py +++ b/seqr/utils/vcf_utils.py @@ -83,9 +83,14 @@ def validate_vcf_and_get_samples(data_path, user, genome_version, path_name=None byte_range = None if vcf_filename.endswith('.vcf') else (0, BLOCK_SIZE) meta = defaultdict(dict) - header_line = next((_get_vcf_header_line(file_iter(vcf_filename, byte_range=byte_range), meta)), None) - if not header_line: + try: + header_line = next(_get_vcf_header_line(file_iter(vcf_filename, byte_range=byte_range), meta)) + except StopIteration: raise ErrorsWarningsException(['No header found in the VCF file.'], []) + except UnicodeDecodeError: + raise ErrorsWarningsException([ + 'Unable to read the VCF file. This often occurs when a file is improperly gzipped, or when the file extension does not align with the file type (i.e. a .gz file that is not actually gzipped)' + ], []) header_cols = header_line.rstrip().split('\t') format_indices = [index for index, col in enumerate(header_cols) if col == 'FORMAT'] format_index = format_indices[0] + 1 if format_indices else len(header_cols) diff --git a/seqr/views/apis/anvil_workspace_api_tests.py b/seqr/views/apis/anvil_workspace_api_tests.py index ae8952fdea..4770a7800d 100644 --- a/seqr/views/apis/anvil_workspace_api_tests.py +++ b/seqr/views/apis/anvil_workspace_api_tests.py @@ -369,6 +369,14 @@ def test_validate_anvil_vcf(self, mock_subprocess, mock_file_logger, mock_utils_ mock.call('==> gsutil cat -r 0-65536 gs://test_bucket/test_path.vcf.gz | gunzip -c -q - ', None), ]) + # test improperly encoded file + mock_subprocess.return_value.stdout = [b'\x80'] + response = self.client.post(url, content_type='application/json', data=json.dumps(REQUEST_BODY_GZ_DATA_PATH)) + self.assertEqual(response.status_code, 400) + self.assertListEqual(response.json()['errors'], [ + 'Unable to read the VCF file. This often occurs when a file is improperly gzipped, or when the file extension does not align with the file type (i.e. a .gz file that is not actually gzipped)', + ]) + # test header errors mock_subprocess.return_value.stdout = BASIC_META + BAD_INFO_META + BAD_FORMAT_META + BAD_HEADER_LINE + DATA_LINES response = self.client.post(url, content_type='application/json', data=json.dumps(REQUEST_BODY_GZ_DATA_PATH)) From e7c42284016943920473d395c26474e623d533a0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 22 Aug 2025 18:13:25 -0400 Subject: [PATCH 27/89] clean up github workflows --- .../workflows/dev-hail-search-release.yaml | 64 --------------- .github/workflows/docker-lint.yaml | 5 -- .github/workflows/hail-search-unit-tests.yaml | 38 --------- .../workflows/prod-hail-search-release.yaml | 79 ------------------- .github/workflows/unit-tests.yml | 6 +- 5 files changed, 1 insertion(+), 191 deletions(-) delete mode 100644 .github/workflows/dev-hail-search-release.yaml delete mode 100644 .github/workflows/hail-search-unit-tests.yaml delete mode 100644 .github/workflows/prod-hail-search-release.yaml diff --git a/.github/workflows/dev-hail-search-release.yaml b/.github/workflows/dev-hail-search-release.yaml deleted file mode 100644 index afd197cf39..0000000000 --- a/.github/workflows/dev-hail-search-release.yaml +++ /dev/null @@ -1,64 +0,0 @@ -name: seqr hail search dev release -on: - workflow_run: - workflows: ["Hail Search Unit Tests"] - types: - - completed - branches: - - dev - -permissions: - id-token: write - -jobs: - docker: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - name: checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.event.workflow_run.head_branch }} - - - name: authenticate to google cloud - id: "auth" - uses: google-github-actions/auth@v0 - with: - workload_identity_provider: "${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}" - service_account: "${{ secrets.RUN_SA_EMAIL }}" - - - name: "setup gcloud sdk" - uses: google-github-actions/setup-gcloud@v0 - - - name: Build and push images - run: |- - gcloud builds submit --quiet --substitutions="COMMIT_SHA=${{ github.event.workflow_run.head_sha }},_CUSTOM_BRANCH_TAG=gcloud-dev" --config .cloudbuild/seqr-hail-search-docker.cloudbuild.yaml --gcs-log-dir=gs://seqr-github-actions-logs/logs . - - helm_update: - runs-on: ubuntu-latest - needs: docker - steps: - - name: Retrieve tgg-helm repo for broad seqr chart - uses: actions/checkout@v3 - with: - repository: broadinstitute/tgg-helm - token: ${{ secrets.SEQR_VERSION_UPDATE_TOKEN }} - ref: main - persist-credentials: false - fetch-depth: 0 - - - name: update image tag in the dev broad seqr chart - uses: mikefarah/yq@v4.22.1 - with: - cmd: > - yq -i '.seqr-platform.hail-search.image.tag = "${{ github.event.workflow_run.head_sha }}"' charts/dev-broad-seqr/values.yaml - - - name: Commit and Push changes - uses: Andro999b/push@v1.3 - with: - repository: broadinstitute/tgg-helm - branch: main - github_token: ${{ secrets.SEQR_VERSION_UPDATE_TOKEN }} - author_email: ${{ github.actor }}@users.noreply.github.com - author_name: tgg-automation - message: "Update hail-search dev release docker tag to ${{ github.event.workflow_run.head_sha }}" diff --git a/.github/workflows/docker-lint.yaml b/.github/workflows/docker-lint.yaml index b928c7c161..67a481e784 100644 --- a/.github/workflows/docker-lint.yaml +++ b/.github/workflows/docker-lint.yaml @@ -9,7 +9,6 @@ on: - master paths: - deploy/docker/seqr/Dockerfile - - hail_search/deploy/Dockerfile - vlm/deploy/Dockerfile - .hadolint.yaml - .docker-compose.yaml @@ -21,7 +20,6 @@ on: - master paths: - deploy/docker/seqr/Dockerfile - - hail_search/deploy/Dockerfile - vlm/deploy/Dockerfile - .hadolint.yaml - .docker-compose.yaml @@ -37,9 +35,6 @@ jobs: - uses: hadolint/hadolint-action@v1.5.0 with: dockerfile: deploy/docker/seqr/Dockerfile - - uses: hadolint/hadolint-action@v1.5.0 - with: - dockerfile: hail_search/deploy/Dockerfile - uses: hadolint/hadolint-action@v1.5.0 with: dockerfile: vlm/deploy/Dockerfile diff --git a/.github/workflows/hail-search-unit-tests.yaml b/.github/workflows/hail-search-unit-tests.yaml deleted file mode 100644 index 389afdc3e7..0000000000 --- a/.github/workflows/hail-search-unit-tests.yaml +++ /dev/null @@ -1,38 +0,0 @@ -name: Hail Search Unit Tests - -# Run the test suite on pushes (incl. merges) to master and dev -# Run the test suite when a PR is opened, pushed to, or reopened -on: - push: - branches: - - master - - dev - paths: - - 'hail_search/**' - pull_request: - types: [opened, synchronize, reopened] - paths: - - 'hail_search/**' - -jobs: - hail_search: - runs-on: ubuntu-latest - container: hailgenetics/hail:0.2.128 - - steps: - - uses: actions/checkout@v2 - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip wheel - pip install -r hail_search/requirements-test.txt - - name: Run coverage tests - run: | - export HAIL_SEARCH_DATA_DIR=./hail_search/fixtures - export REFERENCE_DATASETS_DIR=./hail_search/fixtures/reference_data - export MAX_GENE_INTERVALS=3 - export MACHINE_MEM=24 - export JAVA_OPTS_XSS=16M - coverage run --source="./hail_search" --omit="./hail_search/__main__.py","./hail_search/test_utils.py" -m pytest hail_search/ - coverage report --fail-under=99 - coverage report | (! grep [1-8][0-9]%) - coverage report | (! grep 9[0-5]%) diff --git a/.github/workflows/prod-hail-search-release.yaml b/.github/workflows/prod-hail-search-release.yaml deleted file mode 100644 index 0a2dbb0ee7..0000000000 --- a/.github/workflows/prod-hail-search-release.yaml +++ /dev/null @@ -1,79 +0,0 @@ -name: seqr hail search prod release -on: - workflow_run: - workflows: ["Hail Search Unit Tests"] - types: - - completed - branches: - - master - -permissions: - id-token: write - -jobs: - docker: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - name: checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.event.workflow_run.head_branch }} - - - name: authenticate to google cloud - id: "auth" - uses: google-github-actions/auth@v0 - with: - workload_identity_provider: "${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}" - service_account: "${{ secrets.RUN_SA_EMAIL }}" - - - name: "setup gcloud sdk" - uses: google-github-actions/setup-gcloud@v0 - - - name: Build and push images - run: |- - gcloud builds submit --quiet --substitutions="COMMIT_SHA=${{ github.event.workflow_run.head_sha }},_CUSTOM_BRANCH_TAG=gcloud-prod" --config .cloudbuild/seqr-hail-search-docker.cloudbuild.yaml --gcs-log-dir=gs://seqr-github-actions-logs/logs . - - helm_update: - runs-on: ubuntu-latest - needs: docker - steps: - - uses: actions/checkout@v3 - with: - repository: broadinstitute/seqr-helm - ref: main - persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token - fetch-depth: 0 # otherwise, you will failed to push refs to dest repo - - - name: Get latest hail-search version - uses: mikefarah/yq@v4.22.1 - id: current - with: - cmd: > - yq -r '.version' charts/hail-search/Chart.yaml - - - name: Bump version - id: bump - uses: cbrgm/semver-bump-action@main - with: - current-version: ${{ steps.current.outputs.result }} - bump-level: minor - - - name: Update appVersion and version in seqr Chart file - uses: mikefarah/yq@v4.22.1 - with: - cmd: > - yq -i ' - .appVersion = "${{ github.event.workflow_run.head_sha }}" | - .version = "${{ steps.bump.outputs.new_version }}" - ' charts/hail-search/Chart.yaml - - - name: Commit and Push changes - uses: Andro999b/push@v1.3 - with: - repository: broadinstitute/seqr-helm - branch: main - github_token: ${{ secrets.SEQR_VERSION_UPDATE_TOKEN }} - author_email: ${{ github.actor }}@users.noreply.github.com - author_name: tgg-automation - message: "Update hail-search chart appVersion to ${{ github.event.workflow_run.head_sha }}" diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6a8e929bfa..c8d1a2996a 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -8,16 +8,12 @@ on: - master - dev paths-ignore: - - 'hail_search/**' - 'vlm/**' - - '.github/workflows/*hail-search*.yaml' - '.github/workflows/*vlm*.yaml' pull_request: types: [opened, synchronize, reopened] paths-ignore: - - 'hail_search/**' - 'vlm/**' - - '.github/workflows/*hail-search*.yaml' - '.github/workflows/*vlm*.yaml' jobs: @@ -73,7 +69,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v1 # TODO remove? with: java-version: "8" # The JDK version needed by hail java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk From f1a694f31cab2e485e2628c7bd31aedf8cbfe252 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Sep 2025 14:04:15 -0400 Subject: [PATCH 28/89] remove java from unit test seqtup --- .github/workflows/unit-tests.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index c8d1a2996a..99d4af00ed 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -69,11 +69,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - uses: actions/setup-java@v1 # TODO remove? - with: - java-version: "8" # The JDK version needed by hail - java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk - architecture: x64 # (x64 or x86) - defaults to x64 - name: Use pip cache uses: actions/cache@v4 with: From 55ea461a46520cf45fa60c5115a7172d140be6f0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Sep 2025 14:09:41 -0400 Subject: [PATCH 29/89] remove hail docker build file --- .cloudbuild/seqr-hail-search-docker.cloudbuild.yaml | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 .cloudbuild/seqr-hail-search-docker.cloudbuild.yaml diff --git a/.cloudbuild/seqr-hail-search-docker.cloudbuild.yaml b/.cloudbuild/seqr-hail-search-docker.cloudbuild.yaml deleted file mode 100644 index 91778b7b1c..0000000000 --- a/.cloudbuild/seqr-hail-search-docker.cloudbuild.yaml +++ /dev/null @@ -1,11 +0,0 @@ -steps: -- name: 'gcr.io/kaniko-project/executor:v1.3.0' - args: - - --destination=gcr.io/seqr-project/seqr-hail-search:${COMMIT_SHA} - - --destination=gcr.io/seqr-project/seqr-hail-search:${_CUSTOM_BRANCH_TAG} - - --destination=gcr.io/seqr-project/seqr-hail-search:latest - - --dockerfile=hail_search/deploy/Dockerfile - - --cache=true - - --cache-ttl=168h - -timeout: 1800s From c6e75ffec490335c69d8f549cab9de1e87a950b4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Sep 2025 14:13:37 -0400 Subject: [PATCH 30/89] deterministic order for errors to remove flapping test --- seqr/utils/search/elasticsearch/es_utils.py | 2 +- seqr/views/apis/data_manager_api_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/seqr/utils/search/elasticsearch/es_utils.py b/seqr/utils/search/elasticsearch/es_utils.py index 2381a62b42..94ffead964 100644 --- a/seqr/utils/search/elasticsearch/es_utils.py +++ b/seqr/utils/search/elasticsearch/es_utils.py @@ -206,7 +206,7 @@ def get_elasticsearch_status(): errors = ['{} does not exist and is used by project(s) {}'.format( index, ', '.join(['{} ({} samples)'.format(p.name, len(indivs)) for p, indivs in project_individuals.items()]) - ) for index, project_individuals in seqr_index_projects.items() if project_individuals] + ) for index, project_individuals in sorted(seqr_index_projects.items()) if project_individuals] return { 'indices': indices, diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index ed5ce585b0..64009a6c7e 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -226,8 +226,8 @@ } EXPECTED_ERRORS = [ + 'test_index_mito_wgs does not exist and is used by project(s) 1kg project n\xe5me with uni\xe7\xf8de (1 samples)', 'test_index_old does not exist and is used by project(s) 1kg project n\xe5me with uni\xe7\xf8de (1 samples)', - 'test_index_mito_wgs does not exist and is used by project(s) 1kg project n\xe5me with uni\xe7\xf8de (1 samples)' ] RNA_TPM_MUSCLE_SAMPLE_GUID = 'RS000162_T_na19675_d2' From 84848a6e776ad716d2aff4be193073c487bd6acc Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Wed, 10 Sep 2025 14:19:38 -0400 Subject: [PATCH 31/89] update docs for dataproc + clickhouse --- deploy/LOCAL_INSTALL_HELM.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/deploy/LOCAL_INSTALL_HELM.md b/deploy/LOCAL_INSTALL_HELM.md index 48056a519d..c217bfd3ac 100644 --- a/deploy/LOCAL_INSTALL_HELM.md +++ b/deploy/LOCAL_INSTALL_HELM.md @@ -95,19 +95,20 @@ kubectl exec seqr-POD-ID -c seqr -it -- bash gcloud auth list ``` This service account must be permissioned to create and modify dataproc clusters, and read and write to the GCS buckets that will host your data. We also recommend verifying that the Cloud API access scopes are permissioned for both IAM and Compute Engine access. +- [Create GCS service account HMAC key id and secret key](https://clickhouse.com/docs/integrations/gcs#generate-an-access-key) for your service account. These keys are used by the direct Clickhouse integration with Google Cloud Storage. +- Set those values as secrets within your Kubernetes envirionment: +``` +kubectl create secret generic clickhouse-hmac-secrets +--from-literal=pipeline_data_dir_access_id='ACCESS_KEY_ID' +--from-literal=pipeline_data_dir_access_key='SECRET_KEY' +``` - Create or add the following environment variables to your existing `helm` [override values file](https://github.com/broadinstitute/seqr-helm?tab=readme-ov-file#valuesenvironment-overrides): ```yaml pipeline-runner: environment: - # Stores authoritative copies of your hail search data. - # Objects should not expire from this bucket. - HAIL_SEARCH_DATA_DIR: 'gs://...' # Bucket used as a scratch directory by HAIL during loading. # We recommend a <1 week expiration policy for this bucket. HAIL_TMP_DIR: 'gs://...' - # Bucket used for caching and worker communcation by the pipeline. - # We recommend a ~2 week expiration policy for this bucket. - LOADING_DATASETS_DIR: 'gs://...' # Stores a copy of seqr's reference data hosted in your project's account. # Objects should not expire from this bucket. REFERENCE_DATASETS_DIR: 'gs://...' @@ -117,8 +118,11 @@ pipeline-runner: GCLOUD_ZONE: 'zone-of-dataproc-cluster' seqr: environment: - # Bucket used for caching and worker communcation by the pipeline. - # Must be identical to `LOADING_DATASETS_DIR` stored above. + # Stores authoritative copies of newly annotated variants + # and sample calls from the pipeline. Objects should not expire from this bucket. + PIPELINE_DATA_DIR: 'gs://...' + # Bucket used for caching and worker communication by the pipeline. + # We recommend a ~2 week expiration policy for this bucket. LOADING_DATASETS_DIR: 'gs://...' ``` - Install/Upgrade with the override values applied: From b2329d1609eba983313bce216e3d714f472b6c66 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Sep 2025 14:15:09 -0400 Subject: [PATCH 32/89] remove hail search code --- hail_search/__init__.py | 0 hail_search/__main__.py | 18 - hail_search/constants.py | 91 -- hail_search/definitions.py | 34 - hail_search/deploy/Dockerfile | 12 - .../SNV_INDEL/annotations.ht/.README.txt.crc | Bin 12 -> 0 bytes .../SNV_INDEL/annotations.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../annotations.ht/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../SNV_INDEL/annotations.ht/README.txt | 3 - .../GRCh37/SNV_INDEL/annotations.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../annotations.ht/globals/metadata.json.gz | Bin 595 -> 0 bytes .../annotations.ht/globals/parts/.part-0.crc | Bin 20 -> 0 bytes .../annotations.ht/globals/parts/part-0 | Bin 1334 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 80 -> 0 bytes .../metadata.json.gz | Bin 183 -> 0 bytes .../SNV_INDEL/annotations.ht/metadata.json.gz | Bin 864 -> 0 bytes .../annotations.ht/rows/.metadata.json.gz.crc | Bin 20 -> 0 bytes .../annotations.ht/rows/metadata.json.gz | Bin 1244 -> 0 bytes ...0-8dc59bd2-da29-46c0-badd-a77a850af2d4.crc | Bin 12 -> 0 bytes ...art-0-8dc59bd2-da29-46c0-badd-a77a850af2d4 | Bin 431 -> 0 bytes .../families/WES/F000002_2.ht/.README.txt.crc | Bin 12 -> 0 bytes .../families/WES/F000002_2.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../WES/F000002_2.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../families/WES/F000002_2.ht/README.txt | 3 - .../families/WES/F000002_2.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../WES/F000002_2.ht/globals/metadata.json.gz | Bin 293 -> 0 bytes .../F000002_2.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../WES/F000002_2.ht/globals/parts/part-0 | Bin 129 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 65 -> 0 bytes .../metadata.json.gz | Bin 185 -> 0 bytes .../WES/F000002_2.ht/metadata.json.gz | Bin 350 -> 0 bytes .../F000002_2.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../WES/F000002_2.ht/rows/metadata.json.gz | Bin 613 -> 0 bytes ...0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.crc | Bin 12 -> 0 bytes ...art-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce | Bin 94 -> 0 bytes .../SNV_INDEL/lookup.ht/.README.txt.crc | Bin 12 -> 0 bytes .../GRCh37/SNV_INDEL/lookup.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../SNV_INDEL/lookup.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../GRCh37/SNV_INDEL/lookup.ht/README.txt | 3 - .../GRCh37/SNV_INDEL/lookup.ht/_SUCCESS | 0 .../lookup.ht/globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../lookup.ht/globals/metadata.json.gz | Bin 332 -> 0 bytes .../lookup.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../SNV_INDEL/lookup.ht/globals/parts/part-0 | Bin 114 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 75 -> 0 bytes .../metadata.json.gz | Bin 183 -> 0 bytes .../SNV_INDEL/lookup.ht/metadata.json.gz | Bin 356 -> 0 bytes .../lookup.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../SNV_INDEL/lookup.ht/rows/metadata.json.gz | Bin 618 -> 0 bytes ...0-d213a419-0c77-4952-b924-b6af300f393b.crc | Bin 12 -> 0 bytes ...art-0-d213a419-0c77-4952-b924-b6af300f393b | Bin 70 -> 0 bytes .../projects/WES/R0001_1kg.ht/.README.txt.crc | Bin 12 -> 0 bytes .../projects/WES/R0001_1kg.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../WES/R0001_1kg.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../projects/WES/R0001_1kg.ht/README.txt | 3 - .../projects/WES/R0001_1kg.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../WES/R0001_1kg.ht/globals/metadata.json.gz | Bin 332 -> 0 bytes .../R0001_1kg.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../WES/R0001_1kg.ht/globals/parts/part-0 | Bin 144 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 65 -> 0 bytes .../metadata.json.gz | Bin 185 -> 0 bytes .../WES/R0001_1kg.ht/metadata.json.gz | Bin 372 -> 0 bytes .../R0001_1kg.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../WES/R0001_1kg.ht/rows/metadata.json.gz | Bin 624 -> 0 bytes ...0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.crc | Bin 12 -> 0 bytes ...art-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f | Bin 95 -> 0 bytes .../.README.txt.crc | Bin 12 -> 0 bytes .../._SUCCESS.crc | Bin 8 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../R0004_non_analyst_project.ht/README.txt | 3 - .../WGS/R0004_non_analyst_project.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../globals/metadata.json.gz | Bin 333 -> 0 bytes .../globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../globals/parts/part-0 | Bin 141 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 63 -> 0 bytes .../metadata.json.gz | Bin 184 -> 0 bytes .../metadata.json.gz | Bin 372 -> 0 bytes .../rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../rows/metadata.json.gz | Bin 619 -> 0 bytes ...0-85535ceb-5403-4697-bec1-5eccf7ff958a.crc | Bin 12 -> 0 bytes ...art-0-85535ceb-5403-4697-bec1-5eccf7ff958a | Bin 58 -> 0 bytes .../MITO/annotations.ht/.README.txt.crc | Bin 12 -> 0 bytes .../GRCh38/MITO/annotations.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../MITO/annotations.ht/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../GRCh38/MITO/annotations.ht/README.txt | 3 - .../GRCh38/MITO/annotations.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../annotations.ht/globals/metadata.json.gz | Bin 590 -> 0 bytes .../annotations.ht/globals/parts/.part-0.crc | Bin 20 -> 0 bytes .../MITO/annotations.ht/globals/parts/part-0 | Bin 1403 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 95 -> 0 bytes .../metadata.json.gz | Bin 185 -> 0 bytes .../MITO/annotations.ht/metadata.json.gz | Bin 822 -> 0 bytes .../annotations.ht/rows/.metadata.json.gz.crc | Bin 20 -> 0 bytes .../MITO/annotations.ht/rows/metadata.json.gz | Bin 1182 -> 0 bytes ...0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.crc | Bin 12 -> 0 bytes ...art-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4 | Bin 486 -> 0 bytes .../families/WES/F000002_2.ht/.README.txt.crc | Bin 12 -> 0 bytes .../families/WES/F000002_2.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../WES/F000002_2.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../MITO/families/WES/F000002_2.ht/README.txt | 3 - .../MITO/families/WES/F000002_2.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../WES/F000002_2.ht/globals/metadata.json.gz | Bin 277 -> 0 bytes .../F000002_2.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../WES/F000002_2.ht/globals/parts/part-0 | Bin 49 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 90 -> 0 bytes .../metadata.json.gz | Bin 185 -> 0 bytes .../WES/F000002_2.ht/metadata.json.gz | Bin 360 -> 0 bytes .../F000002_2.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../WES/F000002_2.ht/rows/metadata.json.gz | Bin 639 -> 0 bytes ...0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.crc | Bin 12 -> 0 bytes ...art-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8 | Bin 153 -> 0 bytes .../GRCh38/MITO/lookup.ht/.README.txt.crc | Bin 12 -> 0 bytes .../GRCh38/MITO/lookup.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../MITO/lookup.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../fixtures/GRCh38/MITO/lookup.ht/README.txt | 3 - .../fixtures/GRCh38/MITO/lookup.ht/_SUCCESS | 0 .../lookup.ht/globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../MITO/lookup.ht/globals/metadata.json.gz | Bin 368 -> 0 bytes .../MITO/lookup.ht/globals/parts/.part-0.crc | Bin 16 -> 0 bytes .../MITO/lookup.ht/globals/parts/part-0 | Bin 636 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 90 -> 0 bytes .../metadata.json.gz | Bin 185 -> 0 bytes .../GRCh38/MITO/lookup.ht/metadata.json.gz | Bin 385 -> 0 bytes .../MITO/lookup.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../MITO/lookup.ht/rows/metadata.json.gz | Bin 624 -> 0 bytes ...0-87a9f074-c787-4edc-81ce-94ba0daffd80.crc | Bin 12 -> 0 bytes ...art-0-87a9f074-c787-4edc-81ce-94ba0daffd80 | Bin 85 -> 0 bytes .../projects/WES/R0001_1kg.ht/.README.txt.crc | Bin 12 -> 0 bytes .../projects/WES/R0001_1kg.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../WES/R0001_1kg.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../MITO/projects/WES/R0001_1kg.ht/README.txt | 3 - .../MITO/projects/WES/R0001_1kg.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../WES/R0001_1kg.ht/globals/metadata.json.gz | Bin 320 -> 0 bytes .../R0001_1kg.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../WES/R0001_1kg.ht/globals/parts/part-0 | Bin 67 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 90 -> 0 bytes .../metadata.json.gz | Bin 185 -> 0 bytes .../WES/R0001_1kg.ht/metadata.json.gz | Bin 383 -> 0 bytes .../R0001_1kg.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../WES/R0001_1kg.ht/rows/metadata.json.gz | Bin 652 -> 0 bytes ...0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.crc | Bin 12 -> 0 bytes ...art-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06 | Bin 155 -> 0 bytes .../annotations.ht/.README.txt.crc | Bin 12 -> 0 bytes .../annotations.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../annotations.ht/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../ONT_SNV_INDEL/annotations.ht/README.txt | 3 - .../ONT_SNV_INDEL/annotations.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../annotations.ht/globals/metadata.json.gz | Bin 720 -> 0 bytes .../annotations.ht/globals/parts/.part-0.crc | Bin 24 -> 0 bytes .../annotations.ht/globals/parts/part-0 | Bin 1914 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 89 -> 0 bytes .../metadata.json.gz | Bin 184 -> 0 bytes .../annotations.ht/metadata.json.gz | Bin 963 -> 0 bytes .../annotations.ht/rows/.metadata.json.gz.crc | Bin 20 -> 0 bytes .../annotations.ht/rows/metadata.json.gz | Bin 1319 -> 0 bytes ...0-432b4fda-19b7-4eaf-963f-41f8b545319c.crc | Bin 12 -> 0 bytes ...art-0-432b4fda-19b7-4eaf-963f-41f8b545319c | Bin 375 -> 0 bytes .../families/WES/F000002_2.ht/.README.txt.crc | Bin 12 -> 0 bytes .../families/WES/F000002_2.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../WES/F000002_2.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../families/WES/F000002_2.ht/README.txt | 3 - .../families/WES/F000002_2.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../WES/F000002_2.ht/globals/metadata.json.gz | Bin 289 -> 0 bytes .../F000002_2.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../WES/F000002_2.ht/globals/parts/part-0 | Bin 124 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 87 -> 0 bytes .../metadata.json.gz | Bin 185 -> 0 bytes .../WES/F000002_2.ht/metadata.json.gz | Bin 352 -> 0 bytes .../F000002_2.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../WES/F000002_2.ht/rows/metadata.json.gz | Bin 629 -> 0 bytes ...0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.crc | Bin 12 -> 0 bytes ...art-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4 | Bin 87 -> 0 bytes .../SNV_INDEL/annotations.ht/.README.txt.crc | Bin 12 -> 0 bytes .../SNV_INDEL/annotations.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../annotations.ht/.metadata.json.gz.crc | Bin 20 -> 0 bytes .../SNV_INDEL/annotations.ht/README.txt | 3 - .../GRCh38/SNV_INDEL/annotations.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../annotations.ht/globals/metadata.json.gz | Bin 731 -> 0 bytes .../annotations.ht/globals/parts/.part-0.crc | Bin 24 -> 0 bytes .../annotations.ht/globals/parts/part-0 | Bin 1640 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 139 -> 0 bytes .../metadata.json.gz | Bin 185 -> 0 bytes .../SNV_INDEL/annotations.ht/metadata.json.gz | Bin 1280 -> 0 bytes .../annotations.ht/rows/.metadata.json.gz.crc | Bin 24 -> 0 bytes .../annotations.ht/rows/metadata.json.gz | Bin 1800 -> 0 bytes ...0-2b10c852-02a2-426d-a0cf-961ae060d4c3.crc | Bin 24 -> 0 bytes ...art-0-2b10c852-02a2-426d-a0cf-961ae060d4c3 | Bin 1634 -> 0 bytes .../families/WES/F000002_2.ht/.README.txt.crc | Bin 12 -> 0 bytes .../families/WES/F000002_2.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../WES/F000002_2.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../families/WES/F000002_2.ht/README.txt | 3 - .../families/WES/F000002_2.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../WES/F000002_2.ht/globals/metadata.json.gz | Bin 289 -> 0 bytes .../F000002_2.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../WES/F000002_2.ht/globals/parts/part-0 | Bin 123 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 111 -> 0 bytes .../metadata.json.gz | Bin 184 -> 0 bytes .../WES/F000002_2.ht/metadata.json.gz | Bin 351 -> 0 bytes .../F000002_2.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../WES/F000002_2.ht/rows/metadata.json.gz | Bin 629 -> 0 bytes ...0-5b60e665-6a2b-43ec-b282-1003ad80e87c.crc | Bin 12 -> 0 bytes ...art-0-5b60e665-6a2b-43ec-b282-1003ad80e87c | Bin 204 -> 0 bytes .../families/WGS/F000002_2.ht/.README.txt.crc | Bin 12 -> 0 bytes .../families/WGS/F000002_2.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../WGS/F000002_2.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../families/WGS/F000002_2.ht/README.txt | 3 - .../families/WGS/F000002_2.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../WGS/F000002_2.ht/globals/metadata.json.gz | Bin 289 -> 0 bytes .../F000002_2.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../WGS/F000002_2.ht/globals/parts/part-0 | Bin 123 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 111 -> 0 bytes .../metadata.json.gz | Bin 184 -> 0 bytes .../WGS/F000002_2.ht/metadata.json.gz | Bin 351 -> 0 bytes .../F000002_2.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../WGS/F000002_2.ht/rows/metadata.json.gz | Bin 628 -> 0 bytes ...0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.crc | Bin 12 -> 0 bytes ...art-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd | Bin 208 -> 0 bytes .../SNV_INDEL/lookup.ht/.README.txt.crc | Bin 12 -> 0 bytes .../GRCh38/SNV_INDEL/lookup.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../SNV_INDEL/lookup.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../GRCh38/SNV_INDEL/lookup.ht/README.txt | 3 - .../GRCh38/SNV_INDEL/lookup.ht/_SUCCESS | 0 .../lookup.ht/globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../lookup.ht/globals/metadata.json.gz | Bin 368 -> 0 bytes .../lookup.ht/globals/parts/.part-0.crc | Bin 52 -> 0 bytes .../SNV_INDEL/lookup.ht/globals/parts/part-0 | Bin 5585 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 111 -> 0 bytes .../metadata.json.gz | Bin 184 -> 0 bytes .../SNV_INDEL/lookup.ht/metadata.json.gz | Bin 381 -> 0 bytes .../lookup.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../SNV_INDEL/lookup.ht/rows/metadata.json.gz | Bin 621 -> 0 bytes ...0-7a236e4f-7c20-4944-b7d8-071d2b10a630.crc | Bin 12 -> 0 bytes ...art-0-7a236e4f-7c20-4944-b7d8-071d2b10a630 | Bin 123 -> 0 bytes .../projects/WES/R0001_1kg.ht/.README.txt.crc | Bin 12 -> 0 bytes .../projects/WES/R0001_1kg.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../WES/R0001_1kg.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../projects/WES/R0001_1kg.ht/README.txt | 3 - .../projects/WES/R0001_1kg.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../WES/R0001_1kg.ht/globals/metadata.json.gz | Bin 324 -> 0 bytes .../R0001_1kg.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../WES/R0001_1kg.ht/globals/parts/part-0 | Bin 158 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 113 -> 0 bytes .../metadata.json.gz | Bin 184 -> 0 bytes .../WES/R0001_1kg.ht/metadata.json.gz | Bin 370 -> 0 bytes .../R0001_1kg.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../WES/R0001_1kg.ht/rows/metadata.json.gz | Bin 642 -> 0 bytes ...0-ad3760b2-5a76-4b94-9268-9673bf62e956.crc | Bin 12 -> 0 bytes ...art-0-ad3760b2-5a76-4b94-9268-9673bf62e956 | Bin 240 -> 0 bytes .../WES/R0003_test.ht/.README.txt.crc | Bin 12 -> 0 bytes .../projects/WES/R0003_test.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../WES/R0003_test.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../projects/WES/R0003_test.ht/README.txt | 3 - .../projects/WES/R0003_test.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../R0003_test.ht/globals/metadata.json.gz | Bin 324 -> 0 bytes .../R0003_test.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../WES/R0003_test.ht/globals/parts/part-0 | Bin 179 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 104 -> 0 bytes .../metadata.json.gz | Bin 185 -> 0 bytes .../WES/R0003_test.ht/metadata.json.gz | Bin 370 -> 0 bytes .../R0003_test.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../WES/R0003_test.ht/rows/metadata.json.gz | Bin 641 -> 0 bytes ...0-a35eed71-c848-4567-8937-364bcaecaf47.crc | Bin 12 -> 0 bytes ...art-0-a35eed71-c848-4567-8937-364bcaecaf47 | Bin 107 -> 0 bytes .../WGS/R0003_test.ht/.README.txt.crc | Bin 12 -> 0 bytes .../projects/WGS/R0003_test.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../WGS/R0003_test.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../projects/WGS/R0003_test.ht/README.txt | 3 - .../projects/WGS/R0003_test.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../R0003_test.ht/globals/metadata.json.gz | Bin 324 -> 0 bytes .../R0003_test.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../WGS/R0003_test.ht/globals/parts/part-0 | Bin 179 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 104 -> 0 bytes .../metadata.json.gz | Bin 185 -> 0 bytes .../WGS/R0003_test.ht/metadata.json.gz | Bin 370 -> 0 bytes .../R0003_test.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../WGS/R0003_test.ht/rows/metadata.json.gz | Bin 642 -> 0 bytes ...0-28a643dd-8eb0-4510-8718-6e98b4f4274d.crc | Bin 12 -> 0 bytes ...art-0-28a643dd-8eb0-4510-8718-6e98b4f4274d | Bin 107 -> 0 bytes .../SV_WES/annotations.ht/.README.txt.crc | Bin 12 -> 0 bytes .../SV_WES/annotations.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../annotations.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../GRCh38/SV_WES/annotations.ht/README.txt | 3 - .../GRCh38/SV_WES/annotations.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../annotations.ht/globals/metadata.json.gz | Bin 366 -> 0 bytes .../annotations.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../annotations.ht/globals/parts/part-0 | Bin 330 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 104 -> 0 bytes .../metadata.json.gz | Bin 161 -> 0 bytes .../SV_WES/annotations.ht/metadata.json.gz | Bin 472 -> 0 bytes .../annotations.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../annotations.ht/rows/metadata.json.gz | Bin 725 -> 0 bytes ...0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.crc | Bin 12 -> 0 bytes ...art-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381 | Bin 366 -> 0 bytes .../families/WES/F000002_2.ht/.README.txt.crc | Bin 12 -> 0 bytes .../families/WES/F000002_2.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../WES/F000002_2.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../families/WES/F000002_2.ht/README.txt | 3 - .../SV_WES/families/WES/F000002_2.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../WES/F000002_2.ht/globals/metadata.json.gz | Bin 295 -> 0 bytes .../F000002_2.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../WES/F000002_2.ht/globals/parts/part-0 | Bin 477 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 104 -> 0 bytes .../metadata.json.gz | Bin 161 -> 0 bytes .../WES/F000002_2.ht/metadata.json.gz | Bin 368 -> 0 bytes .../F000002_2.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../WES/F000002_2.ht/rows/metadata.json.gz | Bin 669 -> 0 bytes ...0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.crc | Bin 12 -> 0 bytes ...8-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48 | Bin 200 -> 0 bytes .../projects/WES/R0001_1kg.ht/.README.txt.crc | Bin 12 -> 0 bytes .../projects/WES/R0001_1kg.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../WES/R0001_1kg.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../projects/WES/R0001_1kg.ht/README.txt | 3 - .../SV_WES/projects/WES/R0001_1kg.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../WES/R0001_1kg.ht/globals/metadata.json.gz | Bin 311 -> 0 bytes .../R0001_1kg.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../WES/R0001_1kg.ht/globals/parts/part-0 | Bin 179 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 118 -> 0 bytes .../metadata.json.gz | Bin 162 -> 0 bytes .../WES/R0001_1kg.ht/metadata.json.gz | Bin 409 -> 0 bytes .../R0001_1kg.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../WES/R0001_1kg.ht/rows/metadata.json.gz | Bin 671 -> 0 bytes ...0-4bb6b390-07db-405c-abad-c57b5aa95da0.crc | Bin 12 -> 0 bytes ...art-0-4bb6b390-07db-405c-abad-c57b5aa95da0 | Bin 260 -> 0 bytes .../WES/R0003_test.ht/.README.txt.crc | Bin 12 -> 0 bytes .../projects/WES/R0003_test.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../WES/R0003_test.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../projects/WES/R0003_test.ht/README.txt | 3 - .../projects/WES/R0003_test.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../R0003_test.ht/globals/metadata.json.gz | Bin 311 -> 0 bytes .../R0003_test.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../WES/R0003_test.ht/globals/parts/part-0 | Bin 168 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 79 -> 0 bytes .../metadata.json.gz | Bin 162 -> 0 bytes .../WES/R0003_test.ht/metadata.json.gz | Bin 406 -> 0 bytes .../R0003_test.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../WES/R0003_test.ht/rows/metadata.json.gz | Bin 672 -> 0 bytes ...0-2c340212-6295-462e-b4f7-e067e4839646.crc | Bin 12 -> 0 bytes ...art-0-2c340212-6295-462e-b4f7-e067e4839646 | Bin 97 -> 0 bytes .../SV_WGS/annotations.ht/.README.txt.crc | Bin 12 -> 0 bytes .../SV_WGS/annotations.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../annotations.ht/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../GRCh38/SV_WGS/annotations.ht/README.txt | 3 - .../GRCh38/SV_WGS/annotations.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../annotations.ht/globals/metadata.json.gz | Bin 376 -> 0 bytes .../annotations.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../annotations.ht/globals/parts/part-0 | Bin 444 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 185 -> 0 bytes .../metadata.json.gz | Bin 163 -> 0 bytes .../SV_WGS/annotations.ht/metadata.json.gz | Bin 553 -> 0 bytes .../annotations.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../annotations.ht/rows/metadata.json.gz | Bin 873 -> 0 bytes ...0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.crc | Bin 12 -> 0 bytes ...art-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5 | Bin 445 -> 0 bytes .../WGS/F000011_11.ht/.README.txt.crc | Bin 12 -> 0 bytes .../families/WGS/F000011_11.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../WGS/F000011_11.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../families/WGS/F000011_11.ht/README.txt | 3 - .../families/WGS/F000011_11.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../F000011_11.ht/globals/metadata.json.gz | Bin 295 -> 0 bytes .../F000011_11.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../WGS/F000011_11.ht/globals/parts/part-0 | Bin 460 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 169 -> 0 bytes .../metadata.json.gz | Bin 162 -> 0 bytes .../WGS/F000011_11.ht/metadata.json.gz | Bin 337 -> 0 bytes .../F000011_11.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../WGS/F000011_11.ht/rows/metadata.json.gz | Bin 639 -> 0 bytes ...0-7dd50455-5c6c-48a0-7033-11afafa5d003.crc | Bin 12 -> 0 bytes ...8-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003 | Bin 192 -> 0 bytes .../WGS/R0003_test.ht/.README.txt.crc | Bin 12 -> 0 bytes .../projects/WGS/R0003_test.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../WGS/R0003_test.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../projects/WGS/R0003_test.ht/README.txt | 3 - .../projects/WGS/R0003_test.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../R0003_test.ht/globals/metadata.json.gz | Bin 311 -> 0 bytes .../R0003_test.ht/globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../WGS/R0003_test.ht/globals/parts/part-0 | Bin 191 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 183 -> 0 bytes .../metadata.json.gz | Bin 162 -> 0 bytes .../WGS/R0003_test.ht/metadata.json.gz | Bin 375 -> 0 bytes .../R0003_test.ht/rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../WGS/R0003_test.ht/rows/metadata.json.gz | Bin 644 -> 0 bytes ...0-cbf84037-3354-427a-98a6-b953711ae5bc.crc | Bin 12 -> 0 bytes ...art-0-cbf84037-3354-427a-98a6-b953711ae5bc | Bin 200 -> 0 bytes .../clinvar_path_variants.ht/.README.txt.crc | Bin 12 -> 0 bytes .../clinvar_path_variants.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../.metadata.json.gz.crc | Bin 16 -> 0 bytes .../MITO/clinvar_path_variants.ht/README.txt | 3 - .../MITO/clinvar_path_variants.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../globals/metadata.json.gz | Bin 613 -> 0 bytes .../globals/parts/.part-0.crc | Bin 24 -> 0 bytes .../globals/parts/part-0 | Bin 1677 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 67 -> 0 bytes .../metadata.json.gz | Bin 185 -> 0 bytes .../clinvar_path_variants.ht/metadata.json.gz | Bin 537 -> 0 bytes .../rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../rows/metadata.json.gz | Bin 585 -> 0 bytes ...0-b856a2ae-7456-4db0-a41f-a4541ce89b15.crc | Bin 12 -> 0 bytes ...art-0-b856a2ae-7456-4db0-a41f-a4541ce89b15 | Bin 51 -> 0 bytes .../clinvar_path_variants.ht/.README.txt.crc | Bin 12 -> 0 bytes .../clinvar_path_variants.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../clinvar_path_variants.ht/README.txt | 3 - .../clinvar_path_variants.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../globals/metadata.json.gz | Bin 239 -> 0 bytes .../globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../globals/parts/part-0 | Bin 36 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 71 -> 0 bytes .../metadata.json.gz | Bin 185 -> 0 bytes .../clinvar_path_variants.ht/metadata.json.gz | Bin 309 -> 0 bytes .../rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../rows/metadata.json.gz | Bin 604 -> 0 bytes ...0-c010d599-ce86-4363-be96-0c7b7120788b.crc | Bin 12 -> 0 bytes ...art-0-c010d599-ce86-4363-be96-0c7b7120788b | Bin 52 -> 0 bytes .../high_af_variants.ht/.README.txt.crc | Bin 12 -> 0 bytes .../high_af_variants.ht/._SUCCESS.crc | Bin 8 -> 0 bytes .../high_af_variants.ht/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../SNV_INDEL/high_af_variants.ht/README.txt | 3 - .../SNV_INDEL/high_af_variants.ht/_SUCCESS | 0 .../globals/.metadata.json.gz.crc | Bin 12 -> 0 bytes .../globals/metadata.json.gz | Bin 239 -> 0 bytes .../globals/parts/.part-0.crc | Bin 12 -> 0 bytes .../high_af_variants.ht/globals/parts/part-0 | Bin 36 -> 0 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 112 -> 0 bytes .../metadata.json.gz | Bin 184 -> 0 bytes .../high_af_variants.ht/metadata.json.gz | Bin 316 -> 0 bytes .../rows/.metadata.json.gz.crc | Bin 16 -> 0 bytes .../high_af_variants.ht/rows/metadata.json.gz | Bin 613 -> 0 bytes ...0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.crc | Bin 12 -> 0 bytes ...art-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc | Bin 101 -> 0 bytes hail_search/queries/__init__.py | 0 hail_search/queries/base.py | 1314 ----------------- hail_search/queries/gcnv.py | 134 -- hail_search/queries/mito.py | 558 ------- hail_search/queries/multi_data_types.py | 216 --- hail_search/queries/snv_indel.py | 104 -- hail_search/queries/snv_indel_37.py | 160 -- hail_search/queries/sv.py | 172 --- hail_search/requirements-test.in | 2 - hail_search/requirements-test.txt | 50 - hail_search/search.py | 39 - hail_search/test_search.py | 1293 ---------------- hail_search/test_utils.py | 1177 --------------- hail_search/web_app.py | 138 -- 523 files changed, 5596 deletions(-) delete mode 100644 hail_search/__init__.py delete mode 100644 hail_search/__main__.py delete mode 100644 hail_search/constants.py delete mode 100644 hail_search/definitions.py delete mode 100644 hail_search/deploy/Dockerfile delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/index delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/parts/.part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/parts/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4 delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.idx/index delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/.part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/index delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/parts/.part-0-d213a419-0c77-4952-b924-b6af300f393b.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/parts/part-0-d213a419-0c77-4952-b924-b6af300f393b delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.idx/index delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/.part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/index delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/parts/.part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.crc delete mode 100644 hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/parts/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/index delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/rows/parts/.part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/annotations.ht/rows/parts/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4 delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/index delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/parts/.part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/parts/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8 delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/index delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/parts/.part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/parts/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80 delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/index delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/parts/.part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.crc delete mode 100644 hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/parts/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06 delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/index/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/index/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/index/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.idx/index delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/index/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/parts/.part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/parts/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/index/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/index/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/index/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.idx/index delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/index/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/rows/parts/.part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.crc delete mode 100644 hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/rows/parts/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4 delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/index delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/parts/.part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/parts/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3 delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/index delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/.part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/index delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/parts/.part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/parts/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/index delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/parts/.part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/parts/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630 delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/index delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/.part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956 delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/index delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/rows/parts/.part-0-a35eed71-c848-4567-8937-364bcaecaf47.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/rows/parts/part-0-a35eed71-c848-4567-8937-364bcaecaf47 delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/index delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/parts/.part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.crc delete mode 100644 hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/parts/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/index delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/parts/.part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/parts/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381 delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/index delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/rows/parts/.part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/rows/parts/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48 delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/index delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/rows/parts/.part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/rows/parts/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0 delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/index delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/parts/.part-0-2c340212-6295-462e-b4f7-e067e4839646.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/parts/part-0-2c340212-6295-462e-b4f7-e067e4839646 delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/index delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/parts/.part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/parts/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5 delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/index/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/index/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/index/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.idx/index delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/index/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/parts/.part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/parts/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003 delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/README.txt delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/_SUCCESS delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/.index.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/index delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/parts/.part-0-cbf84037-3354-427a-98a6-b953711ae5bc.crc delete mode 100644 hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/parts/part-0-cbf84037-3354-427a-98a6-b953711ae5bc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/README.txt delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/_SUCCESS delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/.index.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/index delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/parts/.part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/parts/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15 delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/README.txt delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/_SUCCESS delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/.index.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/index delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/parts/.part-0-c010d599-ce86-4363-be96-0c7b7120788b.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/parts/part-0-c010d599-ce86-4363-be96-0c7b7120788b delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/.README.txt.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/._SUCCESS.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/README.txt delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/_SUCCESS delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/metadata.json.gz delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/parts/.part-0.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/parts/part-0 delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/.index.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/index delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/metadata.json.gz delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/metadata.json.gz delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/.metadata.json.gz.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/metadata.json.gz delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/parts/.part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.crc delete mode 100644 hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/parts/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc delete mode 100644 hail_search/queries/__init__.py delete mode 100644 hail_search/queries/base.py delete mode 100644 hail_search/queries/gcnv.py delete mode 100644 hail_search/queries/mito.py delete mode 100644 hail_search/queries/multi_data_types.py delete mode 100644 hail_search/queries/snv_indel.py delete mode 100644 hail_search/queries/snv_indel_37.py delete mode 100644 hail_search/queries/sv.py delete mode 100644 hail_search/requirements-test.in delete mode 100644 hail_search/requirements-test.txt delete mode 100644 hail_search/search.py delete mode 100644 hail_search/test_search.py delete mode 100644 hail_search/test_utils.py delete mode 100644 hail_search/web_app.py diff --git a/hail_search/__init__.py b/hail_search/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/__main__.py b/hail_search/__main__.py deleted file mode 100644 index bef783c48a..0000000000 --- a/hail_search/__main__.py +++ /dev/null @@ -1,18 +0,0 @@ -from aiohttp import web -import logging - -from hail_search.web_app import init_web_app - - -def run(): - logging.basicConfig(level=logging.INFO) - app = init_web_app() - web.run_app( - app, - host='0.0.0.0', # nosec - port=5000, - access_log_format='%{From}i "%r" %s %Tfs', - ) - - -run() diff --git a/hail_search/constants.py b/hail_search/constants.py deleted file mode 100644 index 1e4844d7bb..0000000000 --- a/hail_search/constants.py +++ /dev/null @@ -1,91 +0,0 @@ -GENOME_VERSION_GRCh38 = 'GRCh38' -GENOME_VERSION_GRCh37 = 'GRCh37' - -AFFECTED = 'A' -UNAFFECTED = 'N' -UNKNOWN_AFFECTED = 'U' -AFFECTED_ID = 0 -UNAFFECTED_ID = 1 -UNKNOWN_AFFECTED_ID = 2 -AFFECTED_ID_MAP = {AFFECTED: AFFECTED_ID, UNAFFECTED: UNAFFECTED_ID, UNKNOWN_AFFECTED: UNKNOWN_AFFECTED_ID} - -GROUPED_VARIANTS_FIELD = 'variants' -GNOMAD_GENOMES_FIELD = 'gnomad_genomes' -SPLICE_AI_FIELD = 'splice_ai' -NEW_SV_FIELD = 'new_structural_variants' -SCREEN_KEY = 'SCREEN' # uses all caps to match filter provided by the seqr UI -UTR_ANNOTATOR_KEY = 'UTRAnnotator' -EXTENDED_SPLICE_KEY = 'extended_splice_site' -MOTIF_FEATURES_KEY = 'motif_feature' -REGULATORY_FEATURES_KEY = 'regulatory_feature' -CLINVAR_KEY = 'clinvar' -HGMD_KEY = 'hgmd' -STRUCTURAL_ANNOTATION_FIELD = 'structural' -FAMILY_GUID_FIELD = 'familyGuids' -GENOTYPES_FIELD = 'genotypes' - -ANNOTATION_OVERRIDE_FIELDS = [ - SCREEN_KEY, SPLICE_AI_FIELD, NEW_SV_FIELD, STRUCTURAL_ANNOTATION_FIELD, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY, -] -ALLOWED_TRANSCRIPTS = 'allowed_transcripts' -ALLOWED_SECONDARY_TRANSCRIPTS = 'allowed_transcripts_secondary' -HAS_ANNOTATION_OVERRIDE = 'has_annotation_override' -FILTERED_GENE_TRANSCRIPTS = 'gene_transcripts' - -XPOS = 'xpos' - -PATHOGENICTY_SORT_KEY = 'pathogenicity' -PATHOGENICTY_HGMD_SORT_KEY = 'pathogenicity_hgmd' -ABSENT_PATH_SORT_OFFSET = 12.5 -CONSEQUENCE_SORT = 'protein_consequence' -ALPHAMISSENSE_SORT = 'alphamissense' -OMIM_SORT = 'in_omim' - -ALT_ALT = 'alt_alt' -REF_REF = 'ref_ref' -REF_ALT = 'ref_alt' -HAS_ALT = 'has_alt' -HAS_REF = 'has_ref' -COMP_HET_ALT = 'COMP_HET_ALT' - -RECESSIVE = 'recessive' -X_LINKED_RECESSIVE = 'x_linked_recessive' -COMPOUND_HET = 'compound_het' -ANY_AFFECTED = 'any_affected' -RECESSIVE_FILTER = { - AFFECTED_ID: ALT_ALT, - UNAFFECTED_ID: HAS_REF, -} -INHERITANCE_FILTERS = { - RECESSIVE: RECESSIVE_FILTER, - X_LINKED_RECESSIVE: RECESSIVE_FILTER, - 'homozygous_recessive': RECESSIVE_FILTER, - COMPOUND_HET: { - AFFECTED_ID: COMP_HET_ALT, - UNAFFECTED_ID: HAS_REF, - }, - 'de_novo': { - AFFECTED_ID: HAS_ALT, - UNAFFECTED_ID: REF_REF, - }, -} - -PREFILTER_FREQ_CUTOFF = 0.01 -PATH_FREQ_OVERRIDE_CUTOFF = 0.05 -CLINVAR_PATH_FILTER = 'pathogenic' -CLINVAR_LIKELY_PATH_FILTER = 'likely_pathogenic' -CLINVAR_PATH_SIGNIFICANCES = {CLINVAR_PATH_FILTER, CLINVAR_LIKELY_PATH_FILTER} -CLINVAR_PATH_RANGES = [ - (CLINVAR_PATH_FILTER, 'Pathogenic', 'Pathogenic/Likely_risk_allele'), - (CLINVAR_LIKELY_PATH_FILTER, 'Pathogenic/Likely_pathogenic', 'Likely_risk_allele'), - ('vus_or_conflicting', 'Conflicting_classifications_of_pathogenicity', 'No_pathogenic_assertion'), - ('likely_benign', 'Likely_benign', 'Benign/Likely_benign'), - ('benign', 'Benign/Likely_benign', 'Benign'), -] -HGMD_PATH_RANGES = [ - ('disease_causing', 'DM', 'DM'), - ('likely_disease_causing', 'DM?', 'DM?'), - ('hgmd_other', 'DP', None), -] - -MAX_LOAD_INTERVALS = 1000 diff --git a/hail_search/definitions.py b/hail_search/definitions.py deleted file mode 100644 index bc6aa5b204..0000000000 --- a/hail_search/definitions.py +++ /dev/null @@ -1,34 +0,0 @@ -from enum import Enum - - -class SampleType(str, Enum): - WES = 'WES' - WGS = 'WGS' - - @property - def family_entries_field(self) -> str: - return { - SampleType.WES: 'wes_family_entries', - SampleType.WGS: 'wgs_family_entries', - }[self] - - @property - def passes_inheritance_field(self) -> str: - return { - SampleType.WES: 'wes_passes_inheritance', - SampleType.WGS: 'wgs_passes_inheritance', - }[self] - - @property - def passes_quality_field(self) -> str: - return { - SampleType.WES: 'wes_passes_quality', - SampleType.WGS: 'wgs_passes_quality', - }[self] - - @property - def other_sample_type(self) -> 'SampleType': - return { - SampleType.WES: SampleType.WGS, - SampleType.WGS: SampleType.WES, - }[self] diff --git a/hail_search/deploy/Dockerfile b/hail_search/deploy/Dockerfile deleted file mode 100644 index 4a41eb0e7f..0000000000 --- a/hail_search/deploy/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM hailgenetics/hail:0.2.128 - -LABEL maintainer="Broad TGG" - -WORKDIR /hail_search - -# Application Code -COPY hail_search/ . - -WORKDIR / -EXPOSE 5000 -CMD ["python3", "-m", "hail_search"] diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/.README.txt.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/.README.txt.crc deleted file mode 100644 index 11fbf6bc56e12da1bca755aff3ada162d6aea8c0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9LaM}8v!6H^0o diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/.metadata.json.gz.crc deleted file mode 100644 index 62b1a3ea46eea4775241236a4c752e057065c525..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}6wi*xSPw^L`=#Avgr{ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/README.txt b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/README.txt deleted file mode 100644 index 5e256b9353..0000000000 --- a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2024/12/05 16:23:46 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/_SUCCESS b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 29578f63498a8751f4026ad2e472175650d308cf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}9i;VXQH!<(ne_AkPHN diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/metadata.json.gz deleted file mode 100644 index a7d3fd9f38ebc73cb9cc623161087f30c0080755..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 595 zcmV-Z0<8TXiwFP!000000F_nCZrd;r{1;uL0&R)}g>H4@oKh5sgC1%a0!^-ECK9QV zlo8j^zjtZT;zKfu`ox{(ESIxG>y!!7KvrZw6axk9`RBu`1$vw1S@{OXtXOYwMI{4j z@vJyq2rtJL*tao(yTC%2xmhlVhLuqrHKzk8&85&qgwmRe15p;_f~aH8wnFb(9|NyI z6l3&G=onN&HiVV%E*sFE;b5f2xNd7sp(K1PU}W0>8^e9oj6t7+=$h-zdaxyCFYz!= zxj7cguS${Q+Ev-)L~Bq6@117);HJLV_s$Tz`-Nx&s%&u(xM5)4ce23GAHRI-YL~UgqjhKTij^8M){@n_+$N{@@`W7Q@ujnjcc<3qhWS76qOC*4>h zN4oJ2e%yAu9jN5L{As!Pbo*uDQ+Y=Byk^0cMJ?$YmWeiPEx6);aKruG-RJu$?Uy#} zO#M$D_6(#+nZ5ntC0e4)Bd4PQ5-8QNcoXg4xt5(*~1R7D*u3u;7B$ h9mB_bPN94-VoSTnBE79fj8WQh`oG5N)LbS90053fB^v+$ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/parts/.part-0.crc deleted file mode 100644 index 6bcbbc25221df02188d102f941e8d45aa25fe4d6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20 ccmYc;N@ieSU}Ct?cichf>e8lbs*}C|07K#kF8}}l diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/parts/part-0 deleted file mode 100644 index 6d4137ae107ce1e0a3c9175eaa9698a45ed32407..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1334 zcmV-61$3USJO5F zBqS&l8ajI8UJ=HxluzBdd|(!KZ!9A;zxfNB#G^D>bCvBpA7qWCd=cu_JRwJ|eTrbo zwnTCiy(y~;=D|jSjT__B50byrQrr*h!61O#yLricAc};9hQtJr2_TuL(12V3$xKYh zNXUdpppXg4P!>!A&0Y86FUdC`SG0GPBOh%n;vNQNcKlFkwo)$i?>e;2G(=4bMBJm6 ztvPe3&6W0_&6@3gM(0y%dsj*iZl>AJ$JuhavimYsSOAXCfEpW?jt@!!%aumBh~HaM zY^G~M_^-J%uQ^pYi_LTb&d?!aA6!aWNbEC*(s-kL2M08s!kj8FkaC;$`-dq*H31Gg&e6h%sp&2MAyKD&i*gApUv zN97`7)q#fN!br3NhwhK&<2v-hmD;-`Eg#JNPTje&;~X|O;BDXIcg81~)7ZF~Mzc0= z#EvV_{yra1ZobTaQ&Oh%STQv&k;}c;EN`>OI%YrqN7APYId<|AMNenP0pa!kPUqrl zf6iExfB+_r9?8=DIM2G#OU^Wubk%{)iiA`9IJzYNUl(a((7Y=D78LTN9CzJo{dh{hT`#TMnV@N z6=jjA^|>&sqt?emwGpBwt+2`i8u?6Cr!rY27p9a|Rb2eeW((}mPXk+f+|?`#N^!L9 zADG-MQW5N8326|)Fgk>17w+v(*^NA4dnBL&fYqq+K&22OnHfnzl0q5-KtjWq)wlr> zpeW~@LPj!@CWUqbdJ!snR6QG(fGQE$ zcc&>+t%iOmCQ(S}VSBeRV+?Drxd{ujwrrlQ^9B1p^NH{XyUP&Ub zKfd*6gNfrUK!n8L75%CgDv)6FhuMehGyH}Xj&A~s;cPF%yCF^7dogcMeVI;#7cuiJ z-Y-~V81P0gQ-m80iH)567t8diLKiP}S@0Hrs6@~PIppU}EPNE%3zC3eK??4~gr`&^ z=9-5JVR%`dd^nJETAjw2p^6HjNTziR{XC5maxm&3kenS@h+0SWb6RO&8?~B#5-J{! zb4ye}H2=E6jD(Eum(1YhcF^HoZjg&DVSare)KDAsFs7T2JRu9HEQY7VNU}nIH}8W? sv0AliBUdP6urRVtxlRTauE8d=C-reE4FCWD00000D77#BAOHaX0I;NC)Bpeg diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/.index.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/.index.crc deleted file mode 100644 index 08fcce0cfdbc28dfaf6aace85fcdf8151952cd38..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}D&1=Gg`S5+VZn diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/.metadata.json.gz.crc deleted file mode 100644 index 9b8f8c7b50a2c5e26f7de748ad813a1dbabbc632..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}BK1{`CU@5>W%i diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/index b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/index deleted file mode 100644 index 39358f3ac9605b095b7b8150a26f095624ef3ac5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 zcmdO5U|^61VvVi(e-&h30$B`9jE1NGTo+||hIWCUVlK*_}6LzL*N-96C&L9iz&3HZt)sfB@xvAVpmH4;eMT$DDp0)<5xc lQe@ch`&o@4(6MO}@{E!vDPn5f5ptv-hA(OqMT^e>004hpP1FDY diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/metadata.json.gz deleted file mode 100644 index 278a8722255cfd6bd378c286909ef1ca8f0067ed..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 864 zcmV-m1E2gKiwFP!000000JT=zZreBz{TDrL0d0{uMQiv?b>h}&nix(HED!_)Lycv& z6seMQ8Y9rZ?@%%+icYZmQot}cmoqocjQEmmxB&75+HfVa*~d?3lWCr9DHj7XJ9$5O ze>$1|4M5B3>B$K_Wv5x5Y1n`U$-oc|f;7}_QF-8^ilvWe-tHSf$*m(^Dxn^!nB7>t zWA^3X#CVzsE}y7&B$riQK1#D`JO#W08Hi`f{JD~G2#L9-5u5>`+%v3d zYA8Lmt!jU6bCA3Gwa6UlvQ=oCGj8{@LTkFe?{N<~H3qcBw8s24O2?V_zO&S6-BM#g zlSX402_R(b^xKegPaS1FVn=!R{88qPbzm?oKm6LcD@)xEwdqax>h$%%G*f_4+CoXJ zrqVFY8yi~>u{?6+S|X`hq)D(Ibs`pOL$Fy}U5vv7O0L9|V|(T&v$c&Q3DBbpL;SS(%$-A1}^}#c-UHqWp32iNsg15uxf>QgcsT4WKvnKjc$2%?DXu8fqWUqxiL>n$7UEk}AyNk=j{mpW5 zo3K_&?3*pf3Hf?gtP|lYpMaqQSDM$j*NSBdTFn#Ayf{BkJFv3EpmB4Y?$vSLTc7TD z$5VzI%YC22a&k*Qs_48}o_m?D&ZgvOWlrCDGJ+b4liba6w+dndD=(UNlGK>jzjtGw;b)|AO94g} q0`q_wYn9yK-eOklzrh~LY=)(U7Qb2^-#v>uX!|ds%c&WU2><{BDy3Wi diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index ba2ccefcd48a8ce2b61e721868a24dad1ea4bb8d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20 bcmYc;N@ieSU}9)G#e46qwHA|C`pcUDJ%I-~ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/metadata.json.gz deleted file mode 100644 index 27c89d041da7268e89e69a11b845b20738b128bd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1244 zcmV<21S9(&iwFP!000000Nt2dPvbZgfd5OLcC_hiFF>JhZE2ZG3{)*bt3@bsW4HB? z+F9F0bWr~L9_QlNNeatIvn$OAjpE~T&&lVcuO34?;tjkR!{mkF#r@?nCSlj}z2J!) zyupJP@o+C;jiQJ|M0$^~6>>~Mt>_J23lDdPnBcM504<_Lu=D$5gd|xiu@BX2>TckO zLe3Q3Xc=Wl9Q>F|swih5nK9MtCVn0p1VbUv;UU+2^o68&P}oT>6v2TKh{;gUSV5jl zlE3$ag^qC<8to|K0SYOW#!ZMAXEa37APet9wcy9>ZwMoS6?VIMmZ7tX&C1HfvRF~B zpf#ze2w@Tp6cM{1O(@SdReAKO3?jY`7}~%e_~JYwi1}K!rV&^*8CeOy)st!IqyLNq zMa;?T1v;N*bavv05oMq!$n?D=%@{+!Tfgqb@-=I z@`qFFdK?V#w;~5veId$T@tw1`qh@(y87BUQ@g2eedEqk+D+-LxUxeOxdyk3 z;OhEzG`uctr?6}LvXU|3p(FoKd>O;&kN{C9G$xTkfeg7IrCqLSKO|-d5?}Fnr*EiT zz_`q>sT8Y#^5!KzUNO0j9Z!qN{p7lu&vG`stIFm)I>cMTy1|$G;i9_xb+3>f)H@{g zgBrsGT5~)XbcYlf&T3}!aaA-PUS8Hlc%uRd0pzhJ+%&agYnqS%wRUa?Q3%MC8o~yP z5j<{o-^at*rP&pm-g%I*{X=L^^g;2Q9~90JajpOV?P@XJ_WGa936MSGpa{&#@t-Z; zqrWk%Vm~)8sqa8P-z}J*FEiv@i?~?0z|Vyi@87_EmSH38kFegcsFJ>c^LaWc=3UBH z0sk)Fop#pvY;(O%^&?PkV!B;nLG%WgyDYC^+@g2|;ts&0=mSKVNosZGFx*)43-;++yLY^dHfEJn`vB!aRZC% zD6W-zQ?Gkll$#T{2Hyfs1$%Au=6e=-Eo7d5O3Av~uh&FW-qf!rqfYmHRBriRxTO)c zR7T+N3A{T>D=l2o@1SsYdfGdyoBbsfPE9{(2c;0St6C=d9~43Lw@IajSs_b5RRoDN zpg{il^B<|YmI5-@=?BTb`bx2ntu+-=6Z0iIrvQ}jF;sp*IB&JEGx&L5IEl~KYe|$z z1Tn_5;dMhK$z#)5T2^A!;lNwx|IvfjWL3+llFctNe*g)Fq8k{SADgE%o*=Awg3^K2 z=Q;^0H@iNEndE2BToy$TpdD_`>4Rfv8sLmxz3^(;)uKN>s7<9DDdbn2U1-`5zFen0 zD7)i&Nj&X5MnX-g{!wV~;ORxH-8#qN>BS0nTNrgOS|?{=yR||Xw@|;2&QIHD-Nh&A zZ%E%3h;7IS-Vw33fA!$l6iOGC{o(`{*$6|_4&+kIuQ%vjbS~QMi%wGhCm;%4m4{&vhXt2F4IZcLqK~U1Li_O9KN-L){Qv z_tO&&zh`DB&X{P=pvb{wy(3AI@#U>pYwwd=tQZ)MD(feU*fv~cXFROT!1<1m^NEx# zGfQ$tk>S4}MveeR^EroBxB$&{WOU~$DmFJVGc+_cH)CR8U}JXGcV}czcTChzcTePZ z^$T`4FfunZFt?NjvO^3^jDXe~>KR!j>lqrE89Cay%L2s%3{1>{GA4S4Rt0+QIi)5> zhVD6){OXJh+zhI0jE3fBMn>i)<_1PaKoxm$zdd-G diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc deleted file mode 100644 index 2abb07dfbc0fa2ce089add463ecfb471acea65b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}89^X89KY5@iEe diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc deleted file mode 100644 index 8e6ff63cdd4fafe21b64f08f4027a003aca3dc7c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9MQW{nj96mkQ! diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/README.txt b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/README.txt deleted file mode 100644 index b41496ec38..0000000000 --- a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2024/06/27 14:14:27 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/_SUCCESS b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index e796fa25310482df8fa7659d276a34f135be10ed..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9j8I}!l^5S9YG diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz deleted file mode 100644 index 473ee8ea0f4fc0e02d23f46bf455ad33161c8502..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 293 zcmV+=0owi_iwFP!000000F{x=PQx$|gx|&21__E)jU9#EEPN zDMWd9cAF-p?Ws9A^UaKZwiQ+tC}Q#E03E{O<2kKRPemkFJ8VR}%xI-cK+U>{+ad7Y zR0vNaAQ!SkWRghbF%7QPq4lb$a$Fhk@g#Y4PBqI!8mnBBMF$orMzB|-x&jsO=pI1U z@g6^iJucgOH|fT1Ppf-vEKV^DwteNg-fYmF-R?D!chd)XM3K;6CWn6(nU{J;EhnZm zi>dh~!+bWopI^p4eSm%R${ZaTRdJr|_@hfy%7qt`qo;`p{<*moJh|_MvRjm8ZGllM rd>}}*6haIA!dRWxKoi>|dRZK`ggvZu+@?Zso-^Ga8B;0OI066wU$cZR diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/.part-0.crc deleted file mode 100644 index 3418d733c13b121d5c8a423132d4747cbb87de13..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}AWy{LmHv6PyES diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/part-0 deleted file mode 100644 index 5beb7ba61a4d49690300185d42d94f119f1856e7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 129 zcmc~{U|^^KVvVi(e-$eJGBKF2GBPu;d$=1Im>U}!88e5w2Q!AH7hCD;7pE2$>82!> zBo?QZ6zi8680otQIcFG~>j$~G#{-qcI~C<8ro_8el%y8rCFbat8S9lLr|Bi7S1~X% Y+}q4lkdrr)nHOjm2!QQlU}Rtb0NP66+dM z5-t?FntJQ&F3Q8Glihiv^TkvsYtdqq>kw4BbC8LT0R$-LJSpPJf5_;$J*Eh9CcwBS neol%s2YxrZ!FxJ3O+=oN^A1H!tvW(}>4)J9{jxpC&j0`bO+iqn diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/metadata.json.gz deleted file mode 100644 index eae96f9ae97f56323127e98c9ec99dab2940f8a4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 350 zcmV-k0ipgMiwFP!000000F9E(Zo)7Sgx|%dR&o&uQbE48K}A)2pn!ToQDmG2teV)e zouVl6yLS^3{!YC_o7rzB*|lr3l?t+7uukfNu;=k8o=~#qQe8wc9K?eto;(5Yd=d?Z zJQ5M1#KIOV6arjkL1l*fJ;E0i5zjXm?<)h?Sv4>9PU#I-Z0T(&eC_zbAR%+dUaaNS z8g*IhQra~dD?4elG&%R+*b;n?1^NROTPrO}N4Zjc%1k;okD@dHQ7YYzSw)=M=^bQWp7IJrku_@ZMI?w=7%%{9AD^z(FLYB!fJAw zye$iUXg0F=o+drAG$7igo;TU^0abnczpsVSWVrRx;}K>;m)$>f0rS3huKm|ErxH|{|(>i~98Cq-j1M(|H{e=Pm0C$q4HUIzs diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 4a03ffff0cb622df3b425a67700ba3b3f6bddf60..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}9J$n%q_X(2EBEB60+k diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/metadata.json.gz deleted file mode 100644 index 02f9d0c657ceeb25b4798c2aca2d39a7885d0174..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 613 zcmV-r0-F6FiwFP!000000NqwyZ`v>v{V#snw9q9GNS8N4DMG5I4yrw@Lde)R;5BuS zW2Q2c|Gw*dkpw1#*l&T4ssLtzfiUuE9MA;jTJ4xVWbOP0#A2z%+F(Lb zwYRo$BD9DlB2%e*-P9^(BQR8o?jOt4g%yfOgG&6907$Jwt)xD1^3BgXTf@K~-$pSt z>zn%$ojeq)bdee%zX~A@33ulI!(=wv^t_K1!+aPf3b{VAn}e>fl30~lFscaE_*8U8 z*uNmN+Jzyg*5==h1E+f(977gi8xiqZN`SPVkU=A9Wx%=kjR}K(zc)B7`;lm#m0nnf z!hk-vD^v4B5yIH5rD0lPnfnw(_f&5Xzqt(mx90Kt{S)& z`n%z%V=hMgH#BW(#bKC$u9#3MV@-?d+KlwP|Bg~>LK=250xmUc#p;Api_JSRAbyyV zMqkl4uQOCOPk}sdq0F&M{QhCtdfBH&%#k-YT|Pzuixodl>Ovn@JJ9gnsq(h&w(85B zdQQq;guF-d;-FuArAvD&>#>18wNCq0RaC|6MO&w diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/.README.txt.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/.README.txt.crc deleted file mode 100644 index 17e2f8ae70ba902a4fb797c1a1b41c429b64b6ac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}AVLXWt3{5}5-G diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/.metadata.json.gz.crc deleted file mode 100644 index 55d0cdd18d688c3a7c902325c814d92771301803..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}E6-v?mq-5=#S4 diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/README.txt b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/README.txt deleted file mode 100644 index 22050e815e..0000000000 --- a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2025/01/02 17:23:59 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/_SUCCESS b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 6650687d8b1d7a0980e3d48af24b54da9c0e5986..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}7k)u-^y(5#s|E diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/metadata.json.gz deleted file mode 100644 index 95261c27158ff01413ca0f3071e37dd36a018226..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 332 zcmV-S0ki%eiwFP!000000F_c(Yr-%T{x5ko19d2)Z|m%y1_LX5SScElql>0#Nm8LJ z{qH-m^*&+iOV0T&=hCDmnq@!&@@@ne5YvxITmT;uO6FUrNRXr$@mv7+I0@>5VOA9& zUXBE+z(dINqL5jLohiAi3zKhvyZMS`Qc18uFM2kHRmp`4TLtI&qQAggYGx|B7#!n< z?wBee=t^{G+NAW*#*R61G_=)5$nU&ZgL5b9B+8!xwJ9XKe zd0NeH_O|=7T&=)d{pYtR91Ner6GaLCl1iLOLY4C^mbnluJs8Vx6pTls$MLQ0(>k%Y z{?msu1IwD` diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/parts/.part-0.crc deleted file mode 100644 index 871216cce145dda17e10d163a766e88221cc3d16..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}AX8BJ=|Q5;OxW diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/parts/part-0 deleted file mode 100644 index c33540def91eb8f00ead25af95785a68ee7b3dcc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 114 zcma!LU|^^LVvVi(e-&z4nHat>GcqwSa0VF|7#PMIW~VcUy9P5daJm^7#Tzj&NSegw z<>$pG<|XD-7MH{q6y;~7CYLaWy93p58Jff!nlP|3IG8Np4E{W2$NWIH1)RJf diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/.index.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/.index.crc deleted file mode 100644 index 9e82cfb61effec80dffe2ecec0f03363fe0e0794..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}C6Zee4VX5r6{V diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/.metadata.json.gz.crc deleted file mode 100644 index e05cfb69de77e5d8801db170fbdd78f4463d7999..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}BK`=k*T&5^w{W diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/index b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/index deleted file mode 100644 index 0734b2c39522db842dd14621aa7ebbc3f5161394..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 75 zcmY#qU|^5}VvVi(e-)%I0$B`9jE2VpnV1}%866oI&F36i;ljk|$mlL81C-;3Ddn$Z LU|?hfVuV@%3PKHi diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/metadata.json.gz deleted file mode 100644 index 879fa5f3300aa66735f6a61d98898c8a04706b93..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 183 zcmV;o07(BIiwFP!0000009B5`3c@fDME_+^3ON*8s^%tw9uyQ6FXADs+a?$iNw#1p z{dc#$c^L*~_RaRt7>jod(Rc^4EVE)IpbE;>+PHS7YVrYOk*!Ms8cnMS;35;B0|-#gc~Zoa|Bx|r2TXC=Fz$ul lvm(uf$#WXKr&H5J1I|mM342`3lO-hum+C5#=_HxVlH~&c-!Es= zDW8cMp~S%f9B2VPcc601u?X;04G8kj5RbKmP}y~d=BUgkS8R)}64+R0zCd7p;H6X& zJX=eF>8`R0(tt>Pq~}*gpPb|MAZ8hjPR~m_^ziJ={AJHt`VBQpl@XPvTq#iCGwRGK zj?!1yTlc(s-mHrG;y!Y0)ce{RImX(B=%T_7hPga@(Kt_N0eGvJa;&sr#y0Fsn9>>z z8a-hRIj^h#)V%E0xcOMGBC@q0`kT0{Kk{u*&FTNX9u6kU9ZC%Mo(oeAGYkRmv9#(d zw_%Y_ldQ{e#zNp>lyDIT&zRavk5UU%^()k{CoF6e+BT#O{{9c@t?e(`VFWO`0ssK} C(5>14 diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 02e4874e3f6958a627b9d734fd03d62cad7c9739..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}9Jj=3@5q`=3?-CDH~U diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/metadata.json.gz deleted file mode 100644 index d379d392863f9df5b67d3f6d185cd947c1f308d1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 618 zcmV-w0+szAiwFP!000000NqwyZ`v>v{V#snw4}=yWaJGPtB|T`f@u${P-Nm8a2uS+ zF;f}JfA4it5|S`!RO(B4h{(Ce_nu=PDLxWGcR&I%7Yv?&Eq;AnW)L~VAmJk%NN`V* zG}`C*PLl)@$jJ&zQOY3FL=qfN9ey4%VAHyQEYJ{EKFuaHhozGHNFB>|K?{mfq4-8) zTu^y1vNY$4rvj0=P_9!LFS8LCOGyv+C+f<`WcmP+3UftOQ*UU^6bW&d(_L2eDH|JG zP<<@jb+KZ#Dzu?LFNRuH?)3A+?P9uh{g1WMN&phM*mPA^wVKft*O%58){?c}$1%vx zaoo)&w)17ulnfGWdBV;p6EyIM`dXy52ABLV5{83;JM1s}ohw!@y>cB|1-k35Ec6de za2>#ohxrLB^C%(tLDd%Vo14!iU84!mxJ=!DMe#fkF_*atIeV&(dFUCK_*TXmE>?m! zjB#P4ah!vyl~5s4%u2QTY(`OBOk!i@%t$L>+Q7B3tyT3jb=16*04k7zCf_kPuM1Q; zPlC7{p)#>y%o7MZhusfig}iy_sxcCxRI?*rg9IO-D!6zh;E)yslIXj?Z@kYelabRIkrnZdt_RX@0;Fe;F#-?0^9Q3QPEsHaQJwNj4{FU zCLq}5CVsn#=Q_S+`%b43l5WdBTtvy9VXjM6Xd7y5rblR`&_<|IQco(1XIpiB>@^7h E0Pq4NZ~y=R diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/parts/.part-0-d213a419-0c77-4952-b924-b6af300f393b.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/parts/.part-0-d213a419-0c77-4952-b924-b6af300f393b.crc deleted file mode 100644 index 433649fc5733e5365581e99b364cdf2c1bc8631f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}DJ1DfkHh5=8@g diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/parts/part-0-d213a419-0c77-4952-b924-b6af300f393b b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/parts/part-0-d213a419-0c77-4952-b924-b6af300f393b deleted file mode 100644 index a53545dfd82c2712bb076b9366590423b8986a4b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 70 zcmXqHU|>)IVvVi(e-%^|85tNE4UY#hF*!OjIx;abGJyol=Nww$!o=vv=*|QbVq|3C O1!@2RuqFmZ1_l6ox(Y7< diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/.README.txt.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/.README.txt.crc deleted file mode 100644 index eea7b98bfaabb9f3d3336072da7508278d7ab3f9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}CV?U2zWp5?=#^ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc deleted file mode 100644 index cbb87d8879b009e207acf7fa2dd30e188b0978c3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}E6;F^v%b5yb-5 diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/README.txt b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/README.txt deleted file mode 100644 index 2913b8406f..0000000000 --- a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2024/06/27 14:07:54 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/_SUCCESS b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 4f63d801a1f1ca5b71d01c173bbdded34f3df87b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}8vn@T&~~6fy(> diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/metadata.json.gz deleted file mode 100644 index 1e564200e707b562c91ab2c3845ab9a78baa6f82..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 332 zcmV-S0ki%eiwFP!000000F_e9PJ}QN{g-aRM90j;uoDMYCMJV%;e?P*d%-$TNNFcQ z!oPbvyae3Hs+?Ev$!$X<&4GC2RdGUFOU_4TFum;hqc446I_>sI^rteU){IPL8;c>AXRQjq`_~N7-!hdBS4s zyp*1+eqkiHR6{C_B*vl%B<~JFvdnN? e=#|3tz=#D_W0BUTQLR;FJ?#(6<96)i0ssIfNum4z diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc deleted file mode 100644 index da922333a33f297605c2ae1aa1dcf8aaa0278a92..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}7-b{+t;A5;_A7 diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/part-0 deleted file mode 100644 index 0dfc12b5365badc51cdbb433d5cccb07f0df5c07..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 144 zcmb#$9uHI&?^KkZm=f<=QIcAemzbkpW~^71oTitQUd713 n>1JRQZ^XdHAeh`NE?02v`#jNx6+dM z5-t?FntJQ&F3Q8Glihiv^TkvsYtdqq>kw4BbC8LT0R$-LJSpPJf5_;$J*Eh9CcwBS neol%s2YxrZ!FxJ3O+=oN^A1H!tvW(}>4)J9{jxpC&j0`bO+iqn diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/metadata.json.gz deleted file mode 100644 index e43fa94230e053472621205068ac2368e7081a1b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 372 zcmV-)0gL`0iwFP!000000F9DSYr-%Th5t*QHps+Qn3A`xG8x;0Q`w7@64R@OrAbNB zNh$sByRB_Sw}-uiL_FVNyf1UW%BDF~2W56#v9-5_@Rj3Pj)Y7dd$N|7 zThygK#I!3kR(95Ck*C~)V{7mOw&$5tWpXHfCv>HAkYjTxdK!Ax+_NAz;+=li1Hi(-XYjQvyt`Y{Z^ITJ@bY5!VRrv-bq=ez{psq2k S^SA%2wbnnC1;vKg0ssI_q_^Y% diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index d00b64c90d52a5dce573f5fee9ced1b44ac8713c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}9)~EEvA{q9YFgBwqx| diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/metadata.json.gz deleted file mode 100644 index 42b4aee2d2405270584b4cd601e858e55110fbfd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 624 zcmV-$0+0P4iwFP!000000Ns{dZ`v>vhX0GNHZAG+DzMxUN>Qb1>Y&=iDukT)7`z$> zIc6$D`R_YU$R|u1)p}Ddz&Sq0=Y8!5@PR100pgK6#oz_l?EB4p3t@*iWcdtx;yn== zg}WG^sgxigCJUSjBL<;OB;Fxk;c>qOHZCSG3v38yK23ZY!`$dyXb$OeUM&NQVoMUecjI^fvr#yAxa3Kxo*gKt|8zJ{gP)3VdW%gO^KmF!q zjhvD1O>SjG?fB*)k4_(tqhzs2tg&B5V>L_n_Wi?jHePoJ9}C4s4K*cu~{xH*l`ZJaWwZ6zug)7<&;ChwUDfm;-iUY)Kg0eY;Z1qV?zJ> zy4yc3`w<(KmR?wg&VX*3m5F_!31MMuc(}`BDVrMP12t=mUtN7p(`{4&0 zE_7^`g1uzS^?MG|U*oOJ#ToAnkxrb5^B7E_genyodaSPWNRqQOmR2f8YCKs3m!ey! zI?>cn^G*Q7uc@N)D{}KXpwfOC)OmzbVwd=90L#WX_n_RJ`YH*ri*Knmx5C~7@M>Tm+?E>#yKFypEIgf;72>fZ2f`623N4gvP%XIGKR710JY=* z6R+Ltb_T8X02wYWA}C;lNmGJHPC&8$vW`%p+%ZHu6^72Ch)rw@8%eAeYK*p>lH?Cn KEJC$J2><|BVkm$B diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/.part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/.part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.crc deleted file mode 100644 index 7078a48348cdfe6d46c22cd1b5d2141a0235263c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}CV}b<79=5@-V~ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f deleted file mode 100644 index 170603a856d475e89eb2ad97b5490d71befc61d9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 95 zcmeZbU|_HXVvVi(e-&(BGBN<6`J6*5T$mUg8QmGB!UBVXLW&aek~30+{Q@j4^(+la m@{S66|9<@lpVHI0@R(@YYly zo+biS;2>nYkg^&gqb)aW&5A0AM2b4_!bh!Hvk$3MOgFJVa0YEi_NZ?w!30h1ZI-W2A?M^ zrrt_E+WeWYpX*@L6!c;dFCo2KJJUk{>Zw4I*mg*m&T0Iwexg80{dRT;{pHxUu&3V diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/parts/.part-0.crc deleted file mode 100644 index 581ceab411ec4b01ec95392a47a11df0956f26c8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}6yMii`vR5PbrS diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/parts/part-0 deleted file mode 100644 index 2da408adba902a707a40c7ef734fb23acf74e6b0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 141 zcmbFXD#78dEIB$gx= zr7Ud=8=$9Gml_jU?C8bv}GH|&W i0D+-NyrBsLGXt}8gMqc%u6b#^Kzl&|Y&QcV0|Nk~yd@m~ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/.index.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/.index.crc deleted file mode 100644 index 3b5f783b8e930aa21db2d7858d126e992a06d3d6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}Ct!wSGSU6AJ@} diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/.metadata.json.gz.crc deleted file mode 100644 index 8a49d39d8085dd41ffe821e1211bdc6c6781861a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}AXEINbsO6Zr#g diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/index b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/index deleted file mode 100644 index bf0064cce9c3895da769e8c935bed73361b53c01..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 63 wcmb1UU|`?_VvVi(e--!&fh-0_M#JNQOiYf>jE)R4Kt4ZAn!gex3B(9_05X9I*#H0l diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/metadata.json.gz deleted file mode 100644 index 63e4c6ca648f383ce11aebc2c60d085419c3f4f4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 184 zcmV;p07w5HiwFP!0000009B5`3c@f9hTkP8g&w9GQ+pFZ4+@Hk7xB=|Y%P{n+6*jX z?{4PJO9=d5zWhBj#^Mb_G~R(M%dA)msDg5}Hm=>OntT9RWUEquM$>8nIT!H4bWLvr z2^R`oO}+JX7v*7`lihiv^TkvsYtdqq>kw4BGswor00NYAo)q!Rf617+J*Eh9CcwBS meol%s1HYZq;5{9iCL-6!nNh^nsw3o+ei*)=Z@VJT0001)VNsa? diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/metadata.json.gz deleted file mode 100644 index e43fa94230e053472621205068ac2368e7081a1b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 372 zcmV-)0gL`0iwFP!000000F9DSYr-%Th5t*QHps+Qn3A`xG8x;0Q`w7@64R@OrAbNB zNh$sByRB_Sw}-uiL_FVNyf1UW%BDF~2W56#v9-5_@Rj3Pj)Y7dd$N|7 zThygK#I!3kR(95Ck*C~)V{7mOw&$5tWpXHfCv>HAkYjTxdK!Ax+_NAz;+=li1Hi(-XYjQvyt`Y{Z^ITJ@bY5!VRrv-bq=ez{psq2k S^SA%2wbnnC1;vKg0ssI_q_^Y% diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 3ee859d28626d945787dbad2d30f83702e4c182f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}Bh);PHx|$(RuUAlCz~ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/metadata.json.gz deleted file mode 100644 index 4b5589861446b21bd5ee70902d845fb6705532a3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 619 zcmV-x0+jt9iwFP!000000NqwwZ`v>z{V#snw4~z}E_oxABBW~SpxVPKgpBhAyaoq3 zW-3Ge@BN&RD@+=l`cfXiIp6IZ`v5)?LAO9$G8I0&fIs_jJKq6o5sNIJ;XvFc5=Lwv z<1-CI2q7j546(=uW)g{e%sM;J& z?)WZ@s9E1WWa#Aaw9<=(Ho|_NgjFQnoA*zX*=W=5ea;!D(=d^Xb-{KHs=%s5(}KM0 z#4o2;VwaN};joO_Y99n3Ye#cG_L{A}SA;CWHazr8DISV=Mn09Kl>z7c4<>XSr`yz>E46u{2yV<+(=g4j#G4Yr2dXxR-`spn(hV8{4H5_blP${@2)K;Z zlKU^I=eivW-Cyxrk&6-c9WAY`CGr0A)-G(+o)ZY-pfgjD=w1YB#~oYfhp z8k-MdK>V5t8h%6HyvC!P0SS0<+q^|T~z5^BS zohz^FuB*P@sh6buMaV}qFK*f!|DTdQD0ZjXMHAC}N2R*s<{p4o1M^@v>^`z|7_4lk z(`ehB?x1I{fHiFg7z=tqFmU=bxisN!3$VEO0 F0074CCXxUE diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/parts/.part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/parts/.part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.crc deleted file mode 100644 index 006238b850c6201686ae4d16e8c515f91596bbd0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}BidbD;Ct@yRnWB@X6F7`#N;qQ?WKK0|C#-6)FByKfpamoy?UX^<8iAjz<;Tsr{cSeU zZt)YjP+XRZ=!u&GUwgrUcUH=5%2ZLH4~x?&BZ0qqp@57NarlLcHGHLfw(G|mxBx(3`N)*Kfb8FO6k zV9Q;*--Av5*1xTm?{7XX1C=pRs!C2mmX#4tlqL$=d2p@%Qo!Bq?T5Q-*)NB(m;R@Y zY6f1X%Fh3&iH2MJsw!~wsbc={_8s$|tjBw<3s{;~OHZAcLn0|Hi9pNYL1|I7lJ0a* c(fHtkt$d6}I-iXrTIV(0KN$Xg%`pc60E0yrfB*mh diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/parts/.part-0.crc deleted file mode 100644 index 6e107c2e2c3b76a01082fa35551dcafa479f1e9f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20 bcmYc;N@ieSU}6YjedaTX@#IBy6=p90HOU2u diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/parts/part-0 deleted file mode 100644 index fb90096e108d7516771f7912dd92f06eca9d9cd4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1403 zcmV->1%&!$1pok83;+NqwJ-f(SPR7}07^&L4K0w4B{v%$(HfCN>d1DAu!~N>VuJ&T z5D=^pZL^Q%Z3e#zDs%SjkvAug{saos0Kov@05(Naoo^nLklfE}v=8fL+17TX^}FC& z@}iDJm>KVYCkItuoNfD}5UPgk^7|eQ*F_8At0L1PIIAdjMhC1z}m`+aZ7>(7`F{vaWnviLkHo?TCl7vIl-yMgTREijJG9CaM4u)f4JRFN7dBEvd7RTa1U_Kkn z)}x8X(#UK*qCCi0na;x|(@{BaI2K34+Gu1Ra!?uqeDPLKe`PHh?$y5Bx98{4_^!aa zt{B~W?K`1$uV+h&+V-8_E248gKX1TONh9>@@0VNQwUTS;jJTYG*!E!MEZyX(ytog& zLKRnfzm5@dN1va)z_~%bpGAY(mwFRE^&7jo^z8tIW~QLZ@(w*Jzf`p@DcJM+bp}kP zHA0U9zxot#Bjo&62$5k?FcYjQypG^g3t0B1A=}cA*61K#rQT}NFGB*q#^(>AZ($Ol z@%!-7y?-B^wNgg+V7+Qg8Ur4ZNQ5Hd(SS988bl2dhls>? z{=H3XZY}*RqJv!c@oKgOpA`@2>WJx5Q7?OQ3r?yE#jO)l79=3tWHSc%)q{Ci8qf%cGuC?BB&D4~=6o_lF!*1Yru+ zhD1o=C5K=vqK;D!AnT7`zIjE1b)gXg#+};iI2vbytQ;4tKbiM1%EF%5`Cy5wn$}>; zg53x!Oy}t<;~b#&Sf1R%AE)8j+$ueurgtwSo~I?;7uAaigg&FX^%fi`F@zeK!8=r6 z3u>o3V-~zK7Fc;$QWN^km5?+KG67*2z#>u&sPrxn5=Nq2^OxH&J~+TgWP_hlg+6Ox z8fx4fF1|5ySH9>he**yGYv3RzJ*T;@*~VJUC-dbz61CB$B`zI`gfS=bhe!DumY~r! zL|KyqC~^sO(%S=6Wl=~J;lRApaZ#%w6t)`q@*pNkD|;mv9^{|r_Qq)Ol4!6O-wN0e zvPlt;aU3v~eZ}V1BZexHZ^$r}aJq@JSj%=NR5S4iGq}*Ur$N~O0`{AM4hLCG1Q=8? zYx?Q(hm=H~|My^uuILJXb1^oB2m_=Q8RvudIP>X0BrmxjeK=4$N$_d{=E6BNGr^TT zyoH#yuKkS07M$z!l>;_<*eT-ejc_fKwcXN6WRVOO7p+B4NwQ1x2MquK0000004TLD J{U87V002G;n;QTC diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/.index.crc b/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/.index.crc deleted file mode 100644 index 60ec208eaa1ab84ced18a1894ead44c2c35ce8aa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}8|KIvoxG5px3l diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/.metadata.json.gz.crc deleted file mode 100644 index 680e9425dcc30a7b12e34b17002622c245396310..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9i#YA*%=57z>F diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/index b/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/index deleted file mode 100644 index b5e11439a9087307fa8363696e90a19f8e322857..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 95 zcmcCyU|`SzVvVi(e-(5R8G%e@mgI~g-?K_gjP8t%3=mEf6Qd)e`$0x9Z+|fpV+f=3 VcP2TYYC)K0!A=GSMxcHa-2l`96ny{y diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/metadata.json.gz deleted file mode 100644 index be26a890c03ea9e43d2328858d1616d984907b90..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 185 zcmV;q07m~GiwFP!0000009B5`3c@fDME_+^3OQ6;Ld{JCJt!zDUc^IMw@ol4l5D|J z`tNSNd07T#=Isp7Scf-^$p#Nf%B&~_R8jldSl@1SO(B9RvXvBI(6)v^Ed{)=zxHng z1s954O@j+{m(*dpv)y@P@^UV;bLcRsb&R^dbC8LT5d>)O18L%y|B%V^d+ZaaT!8gY n{G2oy4*WE`F$6j`O+ua_7fd3iRvjT%hH?A?sNs5-&j0`b8DCE8 diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/annotations.ht/metadata.json.gz deleted file mode 100644 index 427ed9f6ee20dbaeb3ac7c671458e501c73f7370..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 822 zcmV-61Ihd!iwFP!000000F_ozPuwsJ{x5wRQip&;9J)7l6BF8o5Qp}HPEjuPwHayB zB`I)1{qJ+qrcHYcn1oR6e6~M3wsSAZmJ1+{pbb|tS)9#hC+BHWP%Z{#a&&ldcs!eZ zK4oV|+br80&yQ)6CK|S&K{7BzgCG^P1%d}HLM+{&d%LRuPwt%Xk_)w=VzIKiX7;5E zVuDng$!jT#S`jwzfT2~pht^#21J4cKPVW97oN~_<% zEXix0h3m!W53pO@3&m=aA|3>ob*j}?`|SVjJ5SS<8&8HFFe|w$YGz~tvxPvMsL0Xe6 zUXu2;^`DZK?iIfM^>rB_s|uKZ$otnj;2YGcr;potFd9`9xp{Uzx!A!?I9mFXsk4B; zpRJuA+?_5>7TeRyoDzp6`Xv zbn69=UMlcL!5F0}O{wypUm2%-{Obg9iRV_IRb4dxqCaRE;U>BE+$Fs1? z=(vQWGON_nN=Qvfs3qbGOO|V>FIV*64dob;u5-2>pp0=5BFu6o1&M->r=%6$N zmTBf=%4ukMP2v}Yol)M5tWYeAH4_?kv@L*d#Ti2Jv#<=Pj<-~K@}bhHh(eBH__q6E zWD!fjYRJe60RBBUg?sW}kYtEEalJx)%ZR2EKjtDsBsBZzY{~VqS_4$BZ%nJS!@GKT zmkMN(%Ft_=KoxgmqaZSCd{d1}^TJPP%3cQ?R$l5t)lS+qmb;lFyqU&HG7S}QJSi+B zZ49a&&XC)~Ins%M4q&*`?JUa?*6*$t>riQ=JBRi_es_ZTxh7COavnexFwmrHh!&|}h);UWXuB}FoMM_p0NwrhhXNZ)dvL;jaAQgoH@k?{8o3Z<6ulr-rbVH7G(h`{_CW*8#5fsJb7heV$t3%|2DUtfD%E%O-;^C~NbU zlMW{9*&QY8VFgKRknaXAB5R>V$aer)ML3AZJ#=&|x@;UEV;xQj#y%HYBsxHJaM*!i zhfbYi-Y20835O9D1RMe$bYbDoIrKg7^_e$7yx}Dp)_o)aE3#hH%isfmd!KLp2raPn zk#>M+9iVktb`WLGvFf47Mpz$Ud-yekFT2w#c&#^4L}ll)`So^wc78c;r+g15EG3r8 z9IE;Zi8xAYBV4iXpfJ6-7*G3Ve=AK&(og5ZBS$W%*G&Lm$ z`RC8Sm+D3e?GcrC>JMGC?Z-^Sq%tgcZ7w=+pLcH1tuD+2Kk<@Y_#%pwb>%{wi>gUo zA4Tyr^t@_Qu@Z6m(Ijx}!>x&`7gHyiU!(y61{|eJ=$jvFsM4RLe7J?mjx~=g8g@3j z--wmur#D^YMpB?784l{mK5Q!B^j<&nZrRQ4qDD>g3_?FVnLWewWuxT_SN zSv*Hl-!b!MBY5=8V01dVpjTvyr>CRK*_w(emf5p&6c9r_{S>HpCFjdb4YzPr)w$3~ySxOcmez1v@TF}(5yIf&$WeAN7IG6hKhO8`j# zBquLY!Yn(9DTYZezze)H8hO)QYFQAOmbKO1dVQczr-ROUO?q{pPe-FM5WKYv!;{ua znkb}~zgvpJ<#y?Td;D&*@@kn~v> zG>m2Sf<-4s@G*13%^fQaVtPYf)*43{a5#t+K!a%1Z2ZuwrYlQG&|=o~T|-uFHzc z#yMCv2_Kp>{Wn60M2sybEKVHx$$Bj|;2nam-Ri_J!GmR)2WRuyiUS=0@0xia5M1VG zuD~^fh6`v1aAatL6b&m_Cuc%TyYA5B6dM*?g$*0?KBEMI!8U}pVQ5w+nh_voaD8#| c*s}pJ3KrM{4FCWD00000D77#BAOHaX02grCK>z>% diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/.README.txt.crc deleted file mode 100644 index 38887a699a60e3173dded0130ae1ea74ec3fea78..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}7jcFj)`)63+t_ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/.metadata.json.gz.crc deleted file mode 100644 index 25bea775a490de8d1cb398d1b3edc5e6d2df5fd6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}DJnboCPe6q*EH diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/README.txt b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/README.txt deleted file mode 100644 index f3c034c112..0000000000 --- a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.124-13536b531342 - Created at 2023/11/27 16:00:11 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/_SUCCESS b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index b2e1a97f659d560faa3ceb155c818449b6b96795..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}C6S{F)yC6Gj78 diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/metadata.json.gz deleted file mode 100644 index aea0d370c4bbde0e536a81cbe075e3d7fc677691..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 277 zcmV+w0qXuAiwFP!000000F{wZPr@)1h5yTaWhRP&n7%n3PbMbJ_#h#qv==ON-O_d@ zbJ>6Q-VIRT$v(B`obROVZNbR`LBOlm=uyw#pYsaU7!p|Ru>rAI(oU5c73(0{BkwJaE*UCF zH*R}gFlar#t~VR>7Z-lX(!23P`fDYiK8@9XBB?Fao=VO_YZi0!O@ZlTazDMsK0?$z z`cIB-jIy{SJ6`mNO8Pi676T0d_;7OzJdE93x-F( diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/.metadata.json.gz.crc deleted file mode 100644 index 908733b0c2b084f62a07d5caeed60d5775841861..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9Lcb6XYw6e$C~ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/index b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/index deleted file mode 100644 index 719af1fef844a8a172ee27e17f7a20e0f33f2947..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 90 zcmXqEU|`S!VvVi(e-*TB85w>sFfg+uXB7FKRbpavXLMwUVq$b;bl2Zs%)}VN=p4ty a;2>deMpX`|M-Zk@u#6+dM z5-t?_h6d}KKFY(Sv)y^4^J*@XwP-QQbqFfmImpDv2m+LIo)mHAKV3K+`t^5sf?`5MRi$lyZT_u3g$9 zS(g5H@5VNSCtsHCobR6Xo~$H0sUZCU>!i_yjh^D@nUVvS>LQZIUexPHgNJ@Rj7OVc z+>Zwlp~S)tENB5PwV=|R`vbxk6%o%j81IW5&{KN`@&ZOaE#&FTMhHHz{V#snw9t)k&s(9CA+f0gw1-s)IrbOu8XV-9 zsSM@6?{h-#Oe&T7QXUXK-|ZadIQT>)-2rjPRC0I*KL2^Q*aNePMb=MnB+eu819k{; zO?@AH2+0aVEa1RQB5_V>hvjh(e3S(+3k-yjPvapCVWHK5=~LX!DM2idT5Jp^BvnUi zAA~}SKq4}gx@AY!c{T#KO3~wEn!2();cHN#QwxC9Nzl@vhR&SQki-kj0s@d+n1?LFw!-Iylp%_HLY}20*MJN0 z7ZbXjPODol`xR;)mtI+i@`3KEm67?N2w}Wi(J)Q1*vbn02dcM--`sqT(+%na^$`KLEhYS$ruGJQ2a8f zD}9*lK*@V`@ql9c}ld5`ABO?&14R=fxK?v%Sk#5CVirSFBgHsDRdEE-=L zP44zuy@A_+UWYZ>!0m=>mUiFs+=1I{cAy{MGJemg+`uq6NN(y+B-Gf!2utoH_#^Q! zPGwv54e6vFCQiF;52F0mg5xI`za{v<2}u6JF3onY-zv9RV!|kHBP8<=9DN7`Zkr6_*R%TEX{R0#CXu@y_008NaH^l$| diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/parts/.part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.crc b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/parts/.part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.crc deleted file mode 100644 index 276c673d03b52964587c7bfa2cdd4374dacd54a4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}CUUJ--3~5h(){ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/parts/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8 b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/parts/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8 deleted file mode 100644 index 817bc3c3afb3628c17d4eb7ea738a128d82857be..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 153 zcmZo+U|?7T#2Q=m|0*on%gpeEm63rZIitw;tP&HWJEJ25Bg1F8;*5!JYwWk$KWp%E z;E7^lbYyhjA=mmka7UkgtDGcD7@zK@x%S@~{Pq_!F@`WYGYTgbm1L$RCYQt)6y@io y#usOnq&}DXV1G@7fs?^aasx*T!CvQojq;yfqIzx0X(g`mnX|Y>#wBEm!ZM$B%^&0fmEx$(L z=>9Q0r6}Q_Qi(H4s0zMAn+wv)gR%TV!el&tnEaG|7%L9af9kMjVA-{@@gELRGHqX^ z0)sD?_~33x{0rXBSiXU+9o9wqDkEy4OP^o1wZ4jF!YvVwD=2)< z2i;~#D=lJA^eciIYZM5IA`0t?QF?V#A_tP%^NRwfN(BPPiw4G-mI#1EED;s}F=P4a zZWjQ7L?Qt!;mK+Z_aoJNbo`NGee9UxfH*pGZj2|^g zwfKL=^{{pmSzpHT=H{?=2C>RFy`7)2KJ>ua4N&>oiN44e!`cbJ2&=VNJAw=!!`{mO zzU%CfK3?pnX(xdYvQ*!VUYe2$kQlQ+1YZL$2tf@`lCTv~~#;o{yvH z_GnwE;ZQ@6f@mZKNh!k`utEut$Tku{)D?vxga{|X|9|%bna>^6((3?GQ-F`ZXipLz(IN8dS@>c4oQ5%- zpG9-p^?*A!(VMOL7^R!I7aoqMvVVcIoS)Sn#@nkl&F!Rog>w(|wfUt0P<63aKgOu0 WI28>500000001bpFa00@0RRB$)IXvC diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/.index.crc b/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/.index.crc deleted file mode 100644 index c90babdbc422b673572e3b65f55db86b82188f8a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}8vMQMe2M5FY}P diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/.metadata.json.gz.crc deleted file mode 100644 index 908733b0c2b084f62a07d5caeed60d5775841861..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9Lcb6XYw6e$C~ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/index b/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/index deleted file mode 100644 index 9ccba1bf4b3e80d4ae6735e6785ab938a9c3e828..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 90 zcmXqEU|`S!VvVi(e-*TB85w>sFfg+uXB7FKRbpavXLMwUVq$b;bQjrQ%)}VN=&ZrS a;2>deMpX`|M-Zk@u#6+dM z5-t?_h6d}KKFY(Sv)y^4^J*@XwP-QQbqFfmImpDv2m+LIo)mHAKVhYr`-M{x3dl;FUgLE53~u#@G&{ExUveJV$N3IkB;o zl~D5Er=(6xOAou`@x3HVKCrDeko|%W+Ez?FKFy|c&I+l`8D!(pbTpaH9{^-NpNz+H zq9%+p4_ok1DF~SdlR1e6iO(oWP<~N7HV%l{bU%lkv0u`NHTqiN!Fl@)3bU;&wb2m7 zQY%b0wKI^AL{~e0e5L%^d)Y)VO?mBdiDXypWs@dP0LNUFrfsk@<_rno-M!a{=yO-6Xn9c78ow(@vTpA?~+)2?z(d=Sh zLA+KtoALq>ymdx~QmbpumbaOfx^8Lu%9hSnP+?%=z>b6OKPy>w3t4?G7lUN&K=nI4 zEppJegxdZ8eZ81WPI}ZxHz-rKjy=SHkAYhB%yrnHPiASCL+1z5!$ragjGjwXt^?{y fp=sWrft(4NFSISCMSp)6du#g#whm(91Oos7xm~m^ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 498d4f224fc7672022f5d7f74d802686d3a32f02..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}6Yb>Nol5f_ZlVB|io8 diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/metadata.json.gz deleted file mode 100644 index b54d201997931940d1ec0f4bb3824945fdbbe0a1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 624 zcmV-$0+0P4iwFP!000000Ns|~Z`v>r$Nv{UZCcXtBP}6sg|P~$noOK@&M%SeE0o2pD5lENmoDuGL{^kfKPsZo2I}z#3A7$Y)EiV z;)Ja;yryvsF=S+hDVA_xwvYtdU5C3(3Vc{5unKGlGhapnn!!}7HPhR?T~G_LL~60H zF(IkiIBAjzEfR^5vDDp;ab9L);94oVx!;w}oKz(bz;u{ts;l|pw>m{Z6lQdl+CD{J zX9*fqlEyS!33iZ7Rwq(sHZ#`W-xq&1XSeqI)6HbK?E0Te;e{M56mroNUF6D&rq=&S zJ*~7RbiR%vP>uU_I~ue*{$QVS2wRGnS4swG=@Avzism*r75}i{>hiLCb-3(Lrg^^f z%ynoU=&HHW*bkZzme7WW@h+AFt04YG^%CP37hm&o8;yZRIhL|%8ie~=%mnYqxp9&x%0QP&sFVrPQ>?DcFy<(mdn*+qHRP;;b3rau9cXH(`5*w| zBPeP76S;XCQ29Is@;E{{v0)@)2pc#15X2(#?$8xu6tG0`6GlEFytrv^d|Tyvu-~0}7ej3Glq&sQ*uw#y7wm=Ga{V6dN3EXcvYy{_J>bms z+YH>kH>WL5qlo*h{4DV>p>hF(WG%T}za{uqRN5LVP9FHDU>K*erTT$#b_|oi^E!RA zKbzqA8_Zu5yx{~Se`S}p*YmseHg{Ms%-aab;x$LEb5&>?%Gk(Gun}X8P_306WXv;^ K=8_Or2><|zGcY*- diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/parts/.part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.crc b/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/parts/.part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.crc deleted file mode 100644 index 866aca1166ce969143f914b947524c132dca1ec8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}6Z<)sF-K5EcSP diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/parts/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80 b/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/parts/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80 deleted file mode 100644 index e25b4f60eb7d545bb14cdc36212eb4ab96679b90..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 85 zcmZ=@U|_HXVvVi(e-&)!GBTtvF|s6Q6#1T2Vq$b>bYx@%N-{D;F#(zGO#6$O7(*DH cnSp{V3Z=8VXy&D@>I@u&gIayA&TWdD0$ZcG>HA@E-ZnsiAUlS)Px=)T=7BxcRHcX5csG| z5HBM@7dQ#YD)5mzrS&zH)}vb`B~snTiDUYpn4Np3H8Z<3wWn6R$Hu}Tuq zv)z3Q{D<|Ee?=bQ-!c<77C$Ta38zdeHkmd=s$B<8CX6~w(h~7 zD4Dje5;p>G3~?`a_r%?tpFLA4_(i3JLk%$&A&$imDA6yZ;#no|Y&(QU0izZ;k5gJV SjajRW%d~&m?G+`s0ssKgVwU9q diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc deleted file mode 100644 index 86bf430a1f89315a10f6162e72f01874df0ac26a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9MR>VpXY6%GUV diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/parts/part-0 deleted file mode 100644 index fe51e88c40df3a0baced17dc3850336707f2872d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 67 zcmdO6U|=u+VvVi(e-#XP85wpkGO~NP8yJ`y8#9Ny2QxBox)~V78!<35SXe%YOlCO5 O3semPU>yvM3=9CAIt)_) diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/.index.crc b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/.index.crc deleted file mode 100644 index 4a7c7cda60bd82e1e99e832d21d7704df973884c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}BK0UdIan5WWJ6 diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/.metadata.json.gz.crc deleted file mode 100644 index 908733b0c2b084f62a07d5caeed60d5775841861..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9Lcb6XYw6e$C~ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/index b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/index deleted file mode 100644 index cb647eb8be0d0d138ac390ff7301bc05753c6b29..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 90 zcmXqEU|`S!VvVi(e-*TB85w>sFfg+uXB7FKRbpavXLMwUVq$b;bT`~z%)}VN=$ypF a;2>deMpX`|M-Zk@u#6+dM z5-t?_h6d}KKFY(Sv)y^4^J*@XwP-QQbqFfmImpDv2m+LIo)mHAKVjYr-%X{x3Ogkcqa#mE1;UZp?$zv5S-v(^m~klai*B zQu^Qb_EU71T|)DGcoNK}Q$7=x`RREel(1>CSMrsP^7uTZrO`6+|3{9V$I&P^F8{y z8nIM56ARrF%a4&*ezFips-vi`v14J<%xF9^C#)h}z3r#wX;b^=YY`5TZ3d#fx$(LN zeT}F&|KHb&(UfuPWb-evFhxH@AFv*%uw`z;2K~;DnjC8~Ks{U~j9~A%RwXibQVXZb dcPJqt3@e7Z#9gsinahHcCm zt(PhU`M-B|jX$AMh?Gk>0Dfm?XP)(I*aP8o1H>T{9>6mMvtM`fE%+uei2DS4;yjWl z_IC+h(IXB0&^*LzC7sVoSNjvT(XtY z7^h-yY~zGU7IRc4Tv}#o9DAd1D+Jv?=BY~~WRV1sIHv)+VbR_NE(NPp<{yprn9Ql} zIl7NxDtoqZ_uyphjc?OZ-7p!uKF>qrpJiygII8J#nJWIjNB$c5RK0&1&xUKO|G5Cm zrou$VS0&UOWU1A8vVx-Ql%s-CP&6S5h6=(EN6G=bT^NFB+~?_N&@uaiGGq{c&7z=^ zGC4kO34|LP4Ow|h^m~wW5VUlaw=mMe#D%YrA zU47N*N{xVq+Rm@?c-2rL6Nz+L@SGVK_Z1(dnt97yj5zQ3e56Jkh6%`G63Tfj>9M-f zBTdhGwnV9b)Lf?kE;V>D)yYhaX+DSn@nPmP`hmWAo1t_+0sOp$v}0ZT%>cJ?vrmnf zDerE&tc?Ia7Qy+UF7#os12ylRk~g`V)R#N;9Lj${KB9SX)86 zkoQa-D)5}4E?Vu@4K>@QXZBjOZ3ivevaV60-InK3&+d4(=eK+M4Dx42`3eT{jt8p# zKoW^7D6s6rf?qNZV^@DIzT-H#hlyj`=1p3jX|Vkm<}S(Ld(^ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/parts/.part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.crc b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/parts/.part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.crc deleted file mode 100644 index 54c1a8dc07f8a9a1142fd826f886a428edd7e515..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}D(Q5&s1M6S@P^ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/parts/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06 b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/parts/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06 deleted file mode 100644 index 2ec21b34838619d6ef6bfff2c186e9685563b06f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 155 zcmZo;U|?7Q#2Q=m|0=9F%gn&S#>l{uoKfU^R*8wxozanj5ePra6=zI*TVub~{#k>U z15Xqaqa&mH4!PFXfjj!_TjeBK!uWJI&9(o|;J3e+i7|xHnNc{gs3bEjF}WnZpeR2t zHNH5rB=x!62m5Ox44e!e8i!vuoIG5(;6Vb5fkLRtbD28EL%cv&fB@JfKsy-#f+R4} diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/.README.txt.crc deleted file mode 100644 index 54a0a746a144369f4ed43537fa98fdf0cc659253..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}DIqlv@P=5pe?K diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/.metadata.json.gz.crc deleted file mode 100644 index 183ab45fa507178bd3b9a90f987081a0161939e8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}9kZ|HXc?0{c4vBsc{e diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/README.txt b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/README.txt deleted file mode 100644 index 635c7500c3..0000000000 --- a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2024/03/07 14:19:33 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/_SUCCESS b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 0847bcb1d5d730040e421eecc3675f7d33b5c8ae..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}BK`;NZ9S(EVlrBt-?Y diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/metadata.json.gz deleted file mode 100644 index d3f8dc9a63f19a008714f99a301caf37ea0ecfe7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 720 zcmV;>0x$g^iwFP!000000OeN8Zrd;r{1;uL0&R)}g>QA^o>~-$gB}_L0!^+JCKAPx zlo8j^zjql?;!>8;^l@u0A!i>XXGa4cGR11h7TFggz<{{^dbQTT&$B!$9$}v?Ztrl# z8v)$o+2Sx1-0U@o4`Tqcz(W{0TP>M|wKFYu2hH5QT_g}^LU=?fX7)L+q}nkP0kKhb ztK%MI35sl=7>~PBH8m3jJT*1gBNz))yw;>v8(A|41(Ol8x{^GFT%&6cN4-^}JTFp} z_)l49m{g85;s)ro@5?&&=<*#HE3xb!Fq07T7reZTmOo-;LQ2BPNC~L6KEX#sI?76_ zvIbuzxpe!*>b)^+e;YdA<;+?z4(XmoIznk|_PKS;XWuaEz!Vxki{rvkmGo|dHYs*A ziJ2go2aN``QYak`jU>vAfdKnq!1opu2aNZRIg77})10~8Nu?*pk9v=&ff8vf9NNO% zp~ELZ=3U{a=2C5A9EbvsR(O!qjy`C@+eS9?C+4K4z{%D#DOIyAwz9(L%r&~a)dJa1 zu1M8dRThW`Xwy6ZcSYGs^6lSs-Yj>)EEAigvd1+tKFU38gI>d~7vZq+;tT6kbSXH9 zEudaxVM)XlkSMAE1firI=LIsf=!Fq>&ol4W#03AdclS;=d35S-v%c1AuJ1x!Ex|j& z)hRcd6pVBd7dzo!r<^PCtU0%u@u|}eHMw6T{{_A@b)*?DnzSEg6CG%h@C)~u_)a>d zBhN|0#Bb6=qt~p#-A){4)GhIsAge(`xzV{M|B!@{HfhN|2Og)$m7b zlO%0CxRyUK;qv0*?d5UVH|xZ6>2KEIoq^TU%Fh4cBQ!JaqpZNjKZlS$7_pVl@kr;pG3Si+HQi6}yd8853jhG+ C`(5(@ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/parts/.part-0.crc deleted file mode 100644 index 4b15876391ea599d70791f5b4a55a5f3c7d74da4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24 fcmYc;N@ieSU}9JtsmXivOrEvbo4X#|Kl+#eS=|VM diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/parts/part-0 deleted file mode 100644 index 9bb8bd7ea9b5db71f80ccb4e54d14a9e43504f7d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1914 zcmV-=2Zi`$2LJ%E761S!wJ-f(u@${K0NP$?5-wntSvT;dBKGZ9sxoTN3zxs(CzGHY z6auZH$8}Po8v$DU&a0OvEm*sSq*d?)3;`Me6#;nbC>-Yd06rO`6kup;?aV*ScT^s6z0mNj9_;Dt=8NGgn} z!nMD5`TdmG^jK;dmRXw&DeF4c%=4p~e%}WBQa40n%bI{gj3O35S_Mc0a6yT%6@w!B zY@>zfvwaq#PpBf(qGm%T1J!tP+QzBA#TZeX%*!YPN+Ftod8@Dl@(sC9NtLcB} zIKunPx9OjRoU(&f;FHiVm^gCt>RB11rqQ?B|ReOq|YyjJ3pErmW3=LRK@ekqn;W8Bo<5d`c{ zinL4t$9`nxo#XAz@AB)u1V)>&eES@yN&Ayu=jmUIvhlwq+q{a8OMrHGRK8A%Em}O0 zup__r@dd_pk0}mdfpMLr*u?=D*SQz~c#}FDB5fKqdk|Z6Ggo=U~vg{I|GHD#;ILM4KL`Wo%Kq3hw zA_WRSP7?;9(@V^iIJ`&1Q+v*Y>)>D3bXhkBNiD3KGJRFnPh2;(YuH%3nA7q2Kl#@k7T*h z9F9O^{~-`GG2+{$2tX2=cOZ2A1*9pA=|cm^5T;DDb%W`vF~0W3dLQChN`K=qA5?KH z&*X9R9EbPAuHm#ANd>gvZ_VB)Vd@T0$nm+wdxc#u+;{s5(}Lu#k&|IU&}LgPFq_EC z)C@tU%0*;7v}OGf_JbL_QON}%hS5`txhY7H78h}`M~^gJ3^ z-cm(0z;fWQpGOlm?heh&9Aiz5O!P58X53E#PW?6?!G^viKJ07zxXE{&50*ZNC?&bs zr^G=&{rW;$Z9wqVyP?%Jiw;Y>MaUDJFlFzcCK*Vki*&Jl6R#Q+yUVb~4OOA+_+l_x zr(tsig;<%DJe7wsrKub2Sfxnx@0n`8z;oI6L{lq(!`-sW*5f9Kty zBKD|~wa^7YXiIO@^g|Wnq|iy}$LU-BD4 z!I@%LQ}06ECAA;#WP4hhMKu-LIdquRDn^}e9Ax5s00G+jK$`gFKVKMY^e)qe!f00005uS=f* diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/metadata.json.gz deleted file mode 100644 index 203b81bb5e6cedc5962d138ff131608bb0ff7235..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 963 zcmV;!13dg6iwFP!000000OeLqZ`(K!{VzVXfLg>((FD4=uH6K662lIH1%iNJsIknJ zB2|)3;|=uRcPNVzMK!yjHBNge! z>OHeB&dGRz(JXV{T;>TATs~3l#q3d|jkrG`(ZBe#f zrRD{-Kq&VbQwq*Pg;iw%Srkz!Ntnt|ga4$lnsRB|5KKc?rvn*lypdZ@?_c_q^R)SnhW?r1wIxy-Cm>1ek|JtDnBkdL>=D-#1^@v)$?WTW=Q? z_d7WMO$<{S(uGjeo=*M+cEU5C;8LO~R;u&UGcu^>?CpOc-j?Z7(1J@vD0AUGc~GpG zBU)*l35Sq1G*h2mkgk~SXg(Ph3N6h9={xgg%ueHj_a{8)GvS7~IZK-3U;0TY0aON~ zA9@6t@$36BQ}vJ_HE=4!a-K_N-15f{NysFti}n0-H0y{5Dt%# z?y*q_z#jL5FELjl+gPeiv-*lt%U-|O7JUL+&+q2T{w_=I=9}9Okq(J&h2%EW#`M<} zvAJB_M)oTu4&@Hygk0QB*U{n@dnpQX^>yhFM6~&POgeW0DR+)>JBxr^rwg*WSzS&4 zY7m+8AD~PXrIH}68Kp0y8Pb-}kGH(W#_()a%1ub43_n<(FplN*v7F-;W<$9IZ;n8qkZ-DjT1^ lRmKex8?)l@6%LS%QUnhWl*Iej&+*!|`UlOaUQGuK000Fm;H3Zn diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index eb7f0f681a0d931a024b76805881195bbdd348c5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20 bcmYc;N@ieSU}BKSY5dr7UAkf8DXwJzJ)Q?r diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/metadata.json.gz deleted file mode 100644 index 05db6f35fdd547d77cbe87f18087369603daecba..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1319 zcmV+?1=#u@iwFP!000000Nt2vPvbZc$G=Nn-Rjak`ou%twQLLM3REpZr$Z=m6DReO z*g3~VbWy(hjq_+bDJ(1PowNu+ZI3NWI7j=3((=&`DT|TsQbwgFLPPjRQ^66T z!cl#Eb5oCT?9Ap7L;0Gi$1$dw5yv68bp^A=7ueEV(eVg&!S}W8UtX?eUz7*yMTEi{noW^FS87;6L_7f*vqWOcVC#qn53P9^ zT81Wy&0gD`9pqUfJ!Fx;7Ko>n!J&yLO)-}_(wS%ehNeNHAClGVezq`@*uI~w??z_l z^-e4~WLvu%Nso6Zm5(HmCXJx=)%?z!v1HLc*k;*iF3M`=RVYlK~kNx>#Qac~E3i zS&-{W8o>cny@$e0;dmHeQUu&|N!iu{vP=T+xdfhux`q7v+r{d9J3ReVldL%LUbVw;Z5y2T)>U`t(EU? zdi*CoB{39Obg@FiLlZnT(9`-9VIGh|!dmPEc{GKK2=1Y$I!jqF^LZ(wjd9uJS()Ba9{pHKbZ zJctaV*qoWk9|(fT+n^D{Rn?bY4mplw1%v+2pa00sWiOQ5Ain~B*S%0hZ9sS`tL;ni zHVs7mAh*DORHr+**J$6yQiA0G*tL{?&M_&~~`8T7{!wJVeZf&7ZO3!CR+&eoS2E9=~INgj#<6-~I%fGezxQ+SE=<@{F*AjgFatoV)(J`^bo zUcD{vgPbabEvJ8UbTrDEuK~gTIT&16@Wd@l{qJ1ncr-p5k52~rN|SmeLcM6`<8`47 dLdSLAOgfQ0nG6OA!KJS(`xl0_Gg8D90005|emDRC diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/parts/.part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/parts/.part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.crc deleted file mode 100644 index 2863e352b2c44597cd9bb896d4c639a134263272..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}89^ym|`&688gs diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/parts/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/parts/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c deleted file mode 100644 index db2e04f107cc88a9c07a0cbd8a65e81d35fb28a7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 375 zcmV--0f_!$0RRAE0{{RhwJ-f(VgjWK0KyJ9G;nP?xUk`WEZ_o3nDaok-#9LPG1B?! z8dj{fE&-{%Ws*`#DHE65nn3wK3ly4K+$71CDa;I2hsVIgZ$yAVLnYfdkh~M-Y(pAl z3kRdgfjo|o>@xY_i-v@a6t4{cKUNNyfXCZO{Hz4pL3aq#R z6<7Metpy7cnixQB>ZmPpF9~M=ai|9V*wU5bkIV6bluf8$fR7~Gan2074X;am95d)x z{E5JjfuBkGTYU_yblsdHVsSWQ4ITpjO$np8&NOpxqr!v+1J?Pou?xtC>xN}(X|~Bi V7!3db0000004TLD{U87V0030*srUc@ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc deleted file mode 100644 index edf58d8f29455d992141ff8579f5ed77506b8fa8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}8A#=w%525-bA7 diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc deleted file mode 100644 index 93463fb7568f4da3e0f101a810e1f7876a1f523c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}DHwmaq%}5{Ls+ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/README.txt b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/README.txt deleted file mode 100644 index 6906dbb93f..0000000000 --- a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.124-13536b531342 - Created at 2023/11/21 12:34:02 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/_SUCCESS b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index f51de5e1ea0daf6ab803da2552f50a975a7aa31d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}6Y;AaMZz5_AJt diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz deleted file mode 100644 index 63920a26b31c12f5df7b493aff86b29d62edce2f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 289 zcmV++0p9)}iwFP!000000F_ZoPs1<_{x80+NHDYs(FWw&064ewUSnsh0v0PE53XO_!5Y1qCtxMDo zBS9);h?q$dOOI)Al?siQd70tbXfGzoy>qf&5^t<@^(s1$fqY^>qpH+0;01ZhO}-9I z&f|ME;l}QcqkC)&PB9F!E^0SIwMW8-RrvFIdtWbMOITCFcOwBh5 z=Cj%D{M`2LgYK;V^wE=1=BLh<4-HXD7oJRpo>m5Yxw-~k-RDBu9mYjvHB$}vKqSSI n39ar6V^vmZT3JWY&*F$BtYM_(G#P^P>}h`hPFstpIRXFxr9O%D diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/parts/.part-0.crc deleted file mode 100644 index e15f050f66fb2f6448ee5f2da74e5e0ba700ce2c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}Df@+8_)751j%j diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/parts/part-0 deleted file mode 100644 index 80e665e3d2c0b3c191f98b7879859f88928485ce..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 124 zcmYdiU|>iAVvVi(e-%=$GBG5vibxh{mrQu`PjfO~KCAK?iwCPp)!wf7U;j~st-;-d zUv`6+dM z5-t?_h6d}KKFY&%XS?%8=ha*&Ytdqq>kw4BbC8LT5d&yO_2{9c3qt7CPe&EL(!>Fi(G=WNWF?!cs0JNU$+A>e%%+4*RUH53@zI566!k zclSVCgu{ui+bQIDxF+k$*%y1yT^P{GD|?fQyU~D7=dCtdu?YKp>^+X3s)JTJwphX{ za+zC&9u@2seC?#nuU y-H+=Yqck87Hw|a8&D>I%uukNuljR4LkP(Kv1Mjh|%{Mf z-^Tb(!w^D<$sCvbh=H#IiMuZ<%yt`KgK`0*Kts54HS}lrePp#mAZM++d77g zsvOI>do0k=EPqPoiB|GAS5njPp#Q&(CWCdi_r0W;b;Cd|Ru#P|s1hrfm6184lF$L4 zlFk6@=VX@KAOP8z{=?8~cY0p6WDtMNLv|42A?+7rPzzdWaLWH-fMeTT=Q!;tRxC}u zFb{ z^^}w|LcXGTacCbrkvpJ3a0b-Bvx2K3(M{i_>g%RyZUR{MmmLuiQ8_qTfL;Z zYGD5X1`K8dx14}rXY2H8b*x^;ww#tzFEzrLep-nOKio1jcS;SlLx~M_6OANV4OL3& PUPVvVi(e-%tuGBPkSuq0;`8GiiMz{14o90CQ5&W!9qscFuMIXTP>%wKHI df3`XQ*j|)}kAaDofrWt=XaWd;O#vFg007Gq6wd$v diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/.README.txt.crc deleted file mode 100644 index f9b6a2f01c5f72f046bcb43657ed33333cc334b2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}Bi~P4@}_6VwC4 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/.metadata.json.gz.crc deleted file mode 100644 index 16db99e30cbddc3cbbbf4caf7d9407ce92ed36cb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20 bcmYc;N@ieSU}BhC$)a=m_=;CK-CSD%J^%+> diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/README.txt b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/README.txt deleted file mode 100644 index 4483c2b2dc..0000000000 --- a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.133-4c60fddb171a - Created at 2024/12/04 12:35:22 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 65da121be0760fd04e2faa82989e28f8894d8bc7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}A_o>z2EyC7B-pBYgy{ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/metadata.json.gz deleted file mode 100644 index 52d10fd556a77c571e4aa762d4425c5e0e2ce588..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 731 zcmV<10wnz(iwFP!000000Hszk9Z~k|xHSy1vx7U3rNAGRLtzt z%IGKHnB}z;ows>NcVMj4%9RP>g;YCcdbrpqyRCW~WDaW4Suc!tMy_0@4sVRfaI<4&ra{**<-q{1%AasxqoRvQG7QIevFtni3| zpEW2>t+ZsMpz3mhh|pIteAILf$tCU=o_$~BA!XKrL85dRX<8D)T3>5qF8PjG1d~;C zQO3BI>>k#%l!vaE__MK={Wutop!a)kK-IW2KdQK#pI1mCkunC)j56x59cb?~kP~ki z(q@cH&rZCrvyu8-jWhscG@5dhQ0OfJM77A3ZDm2}Io_4n*)ECq_S@kd{?dEvc-!Lw=Wldd?sSz~>-CH3PwkayeB!Yj zov1(ILNSW`O+eF>3Jm}qlY#t9D9@DWoY9+UsTsV!=RDAuNL~hjxlu&Aaz;`H<>u!U zZb&?Y(SF2XuOnj#8MF`y$Sc|r!~R4$rbOdB2Xm6~A6$5)*wouCn88=km(}v!<%eY- zN)!Gl3lXF&3(cRYPL#B9&b9nS3D;Lw@2|(O9~Q+C`ja4^r`>79_mK5D^NUZ{>2Fr*YL7qOlIs N{070X(W7t*006yrVx|B9 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/parts/.part-0.crc deleted file mode 100644 index 3e40b058a76736e0b6abeecc76f2dbe11a360f43..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24 gcmYc;N@ieSU}7lz*xhhNs_Myfr?(HdZd_9Y0BlqWa{vGU diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/parts/part-0 deleted file mode 100644 index 2f32df4a28bc51e7d22008a080bf3d6d066df006..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1640 zcmV-u2ABC$1^@s74gdfswJ-f(0SzrO0LoDW5HX<8**3uI1Il|^H```XB@#+W+lEhE z?{PTKf+08FEFH>uMqzaKxmZ1rNB-l%=0gJY0Q3Oj0B#$+!&^JPN=WPaM%g~~3!`st zEu4|L_Zxj~PeTnbvRBrp3SKM9a29wJ=-G2Z)?mGQ+?uVyO&kKzzd5?P@MO8Jj_bN0 zbcgObvhZw3HXl|6$%bU(sTH10i3Ev9>LHTV!n*Y-HEI)3?nQSu9&txU!s& zr{nqWgij<*^EBfLzxMqSqevK7LRu*>3DY>82s(IfxGt5CLzarI-L^8b|Bh1t`v1!Y?0i(+x2r7oOswEWt7 zOe$?)7qpa?x?o~b$#R1AH^(6+l`yUfmu7`d3=7I6@+MFwk(@x8P7 zzA86b*I9nuep|M+x2N$#{zo^y^f|;MwH9XB8++S87T0B|paAlmf2%fXTMLo5 zx^z_mj0?{EAPe&jJ1V`eT9+N%dG$U3Ces>WrvbnEwD4Qug$O3WJkYA}x`F=`uxu3q z*_L{=MqhcAcCAT05exhppMQnEh71gj--j9Q-SgnA&7vOBFI%N-Kl*X5>fnaudS0Vt z+17TX)dSgDG71YB7*P{rxOYh-zh9o#`_8hva(MJwe)^Xdj4fIAg)Y0}w*Q%sC?)Yx zIbw$g<|mAHqPtcHs`5(TR)8`s*HgFO7T`du(hKLubro<;T-edNiu~=-5Dt%vy86Dj zC@wnfw`E+n??+GHT(mSs$Z)Ctz=emYK6|b&I0SLf)*U__O z$2-4!#Q*B$p3dg3-wlBUsKG?VjG37k5s@SXNjh-^1aTBXW;YTbQ!z|~4mv|4Q(KtA zv>Q$03Gl-&L^528Ta4YN0N+;C^bwxAX#Hh+>c~H-NaZff(h*%9;ZRJvw>X^J!A3A2 zA>YXfut3bVvIljqP-Y%oU^ld1i;nHxhnHV=Wiq1IfG`;IPKU-VsCAdso*R>t6y zDkIPdwcQ*VQXRqMt7StO7Hyce(;q2lVHn?yoLgPW$-v+)zjhD?!<)_zD_W&bU=cH~ zH|D!p>?-L2?y(uTzj#W4^PX3|;dvNKmgaWNEJO4cqcF@v4?(@tc?JxsyPWTKAFhG? z5j%$J`P`@kHKxT7K$FY_N_R&e#m?}T*#MI9w55Rub5Hz;$Y!azxiu6+5hm+YDjGBr zh^!>N80E$x;$J-;uS@JqQWtLG8_P6OyoLm192s6KuXJ_8k8J|j?2!mYpK*i(n+}C} zv3I(6CMbf;EEwydN7fx-2bd8;*)FODj6LCIs^<NVaHnz& zB?P=p-3wX?7v5LM!ZH|XJo40t2uR10Cp=JjNVJ(yiEVvVi(e-%>KGBKF2Ffgz&uq0;`87}f?Vs>~Px2xR0wdcw-Y z7{ciMmhteh-9qjcnHoP{&}L$EXLM}lY1@7g$P8h0Zxdi+NT@JqPE#;upJaJXGEG7e TXoCpM7LkPv42+CG9-1Wp!}TJ> diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/metadata.json.gz deleted file mode 100644 index 5b44f36c6f25e26361bcff6cfbdc5dbf74c1c5ca..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 185 zcmb2|=3sz;(7uCQhYWby?yIV|$}Q_NtQL0U=j!Sb{}_-cB zFW9Z}{qdX}|Ln~xmOELt^G%L9y5duBx6dAa1&2y`55fI^+Iwc+PxQ-CI`B)quCI$n lyv};LA&ZrXuhX>|qB@H@Hum>T$p4l2gDIuPaoPo-ivfGmO=E9F)TMX<`ZZcBF&ZxyY$lMu4pWSV-{ zv2(V=(nbC6H%{%uNkc(MsAS&9WBZx$%V5PgkS9=@Nim2|&xXfeqrsXo-Y^GWJ|BNR z8V>(HNlw44(sX%rc0>o!K*0(Why*lIz)3;P8kHL^su;RN_hwT7N*0!Q$+%on9?y*` z6Z29QDK((uIhY5>N+{TU+5WsW-#Y3NkE0b!fZp-rgtcV9DgZaTI70EN|WsbGb4A-UU(wZzk$> zQz&h4A#tZb;_J{MV9buJ9Zj;y?^!AZjY%$$mu!$eI zOiCRg8~_LX7%BO?sSirxEBpJ`_q*|W`1O<5>x5F-@Qm!KpM8oDKk6Hc_lQwpNW*z9 zq>e9yIX!vT8TvzXA{E3_E~yzFA2?DN!>5SuhsRja&Q0E}<06eKxvkX-?}fp@v*3=i zaHV?yU{~VJOH8x+M%c_GPv?dz?WC_5)qt^=ZKTU(&J1o^x6qhp%1njH2T|tk{Qmr^ zU9RZwb{7C8IL9tQmidv)<>X)mFJe&(+J>mG$pg!+4bn(+UIE6e2&Q|{pqqs z;s=<3ka;M{nWt_VqgTK=3~^ICaUX$SpBuToBj2xXzm2YHyD16-H%Z-U-;@U$9SMAQQ4?uW=>s!S{3eV2cU{B<}J_ya~9cq*pYb+0<9Y*b+ zr0UDsZqQ5<+oJ`|@Gt%*B?l^^rfv7Y{O;84T7h>1&jv6##H1==mJoL~UB_^Z$p`3f z|M1MTZ4c;VrDzUBO11I`Ukhbv+ugW zYs+%#RlGiB0l(*q+gYE8aznz^dz$dwT6bFbG$OBhx5Chcr&}&?GD1G?3F*6g*x=te zowWes1DCMJe@_+%hRs$2Z^vtU3PX)Y7ZXoy@x__QypRHf(S!J9uzPG)p8|IpAE9^m zld)Hs7m!r%*!uD6^|#2Czu#|HDWg$=n_CS*q?)@z2Qx1kSCXzXum5oRX#FTz*a-|; q9%`nuX=|m`MNSOgT*3y@L5#%(J4<`}>iu1rZvO{1x1QJd4*&oN$aabV diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 8ff1919e93bad55500692a1821fccc598bbe36c0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24 gcmYc;N@ieSU}8A=V^Z_xy=y0P8Go_Ti|Hu=0B%+Zwg3PC diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/metadata.json.gz deleted file mode 100644 index decb880869102400173dc392b505826a5571f9d9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1800 zcmV+j2lx0NiwFP!000000Nt5wQ`UFS@X1W2jBg^;vNDKLcLZmz?4ge@m3 zw&b%EO3L`Z-^wqtBr72>(=Y9G+IY3v7iqP6;yim4W!MY6GaBP39ADp_&T||MJ>Lr- z@Y)N$c`1ul9G+2{;uLf54u&EY<47uc!E!YD}2s5lswe&afrj*9}VzqOioez7?C5WdA?!ATsn!;h#S zh>B1XJ;9VfDvu8R1!WnEL&~Vko6rUPN4Y2vp~7(cGvpB~a4?;v3{`G;RA5XEr2;P? zl2E|XfJ2p29oSvzM=7BYkTIvu|11t_Eg$?RDTxHca)rc_Eife!WN0i=H$B&M)>W3R znH6NvuF0LhU}-!TsNk)_riqAiT*w7=WZ?7FeMW@9MK{uJ7sMS`8_^po{rH~pBoB+fuNN7bHUhkI#AmUQKM#WPExW=1cIPRSYHzt6-*5(${-Jk)bx8 zg;$qXU&sG+4#qR>^CTR{-`hOk=`$Ak6U&UTjHt&8ZDQFT2K>Y_%&h=4Zx@}l_LJ_S zjWxWy4nLh=o{Z1Cxic8mZE3+M_plYwLu?r?~R;X1~tK=$@FsYQ)# zf8mN`srQJtJYt@^r;R*=%d~&CoqzO=s46t=d262$`$p!D%qh zERnZqYe6uf5-feVG>YA|QJODd;h^6_6v4)(t~G{~B?4o!j}i`-WD-)efWKgcu@oaJ zRl|g&vd_70qPl?V`^vP3p9F@Op)*#=kSJ`5y}8_)3sre4{gvDu3aJTZUTye^aCwh` zsiXjq2pKC9bpfeSwh}Darm)hb33-0-LT2&y!ACfc_){5t7^KsnB}c>`bc6KZnsI}RMZNMo95Z+Z6+7!Zz7hJ zteH@Gb*fYRV}5ga>#GBvu7B`;MdSyzN9g9J%7jm>|i{q3q(=v}pL8jL%NhqvY$yhOZ5>K9~Rg z@#%W9JUD)*$hmd`if~p8Y}V92ysNRe*3zlCkyKE;0~1>~Y}4=t5UxYCG1w;H7Xesh z>LgzseEZnDfqGrgt4FINues0~@EQXOcO9VX;#@c8Iw*H5^=p+aA+uNNW+T0kj6snmB9FtP^HCj?~yoQDnzpn;FtVSc4$l0BiZI(e?E{J8X6N z?4Z@btJ`taq}2`-6&?1Ls*6@_NW-W-JZ;*FqEj1Ad+sj!ewVGg7n#&(3_6UqVJH?g zP_!9sNwj&aK~#Il+XwRoYlJ$5>LL?Dn-T5C+HzqT4X6T8gC=#DG|BTNH4{0Vv}wbp z6EinZrrG8V#(X7Ry5Vv=SH6UmPO97rl^dB-$H``A4Vu(t3MLz&2~!(U(n*pQNZK6P zcyMB*gCZMs2jXpENDCnw8#Y011xSk@8xuA@Zl%Y1&hGGNG>A7GLN_zoQEWj*BY>Cv z^-xi{sz|YUlAw6MN)lXH?`zMeCxhYfNjKzs(UPRGrg8#|_y9%>rF$uylb@h4IyyQS z^{4&IMO;mNzpVA4da(_ecBjVRTUaj`;l=(FHz%uj8?t|Lk!| z0y44sS4a3zmoRvZmc;T*;G1_l2WFFgeau1viu(h_f9`p9RL8|!EfFG diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/parts/.part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/parts/.part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.crc deleted file mode 100644 index 57325f97f811abc958805af08bb4804721e61837..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24 fcmYc;N@ieSU}Cr}kaXoLcja{6FJ87SiZaRoTu}$O diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/parts/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3 b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/parts/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3 deleted file mode 100644 index c2db679b34bf5abfbf34860f84b5e7879121f80f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1634 zcmV-o2A%m$1^@sh3IG5owJ-f(B?^)j20-pcd^BnOKK z@B{2s@*0Coz`d@pu1(h3PngD2rr7V#y6#@tE4M_q?FU~Qh_wI-`Fg*u-7*Cw0XG35 z0dM?!-I1pQ!dSQ$YoCvDQPbz8&$iFvi@8P$EMmqyjSrGGUT|cNT=#jJsmjUCwE5mz z-9Z?sD9slcJL3Xi0prE#^S{ymlDd@$&eDH6CV~!|fB}mwVo?_xY>ji7(@)pI4cExS3F!y-F$n3^1A6W!@f`)=K&53R_MDdUCoPB;Eo4e_BQo8|bw zHThD*8W94eYeF)bAuN$1960CvuX220|4*m<=wf#Z>Rg2<+OT7VHCz7wuW;P?%8C9n?G2tfW-aNu zE!oelH@TNK$~Rxplm}tk1+$&o?7A&2ykNFrv7Knd=W|-8>DIU|uB;|Ej;Y!>uH^|- z7H9zeYvN6amLM+#Peb>Xd-NfoKoY_@-S~4e(Ue6ywBp6T-Eiu0L*u2)>xo!pP7@O_ zw;P0Fd~FI4vQNfxDBJ<(wj;^vrp8Rkc=b+T-=1-PqtWwl+wb2H}?n1A{#YBkzrMzE=Zzwk}pX;KkpI`}KV8I2G^S-w)`%0)o3< z{Kmf?Job|LumX%z!FSgygWs;P1PqLIFc3^Lrd{ArC-!Cj98@b}`zpRVV7dB1mSgn?lr98;pC`2P@0XkLOKnc-mBbU<>& zLILCYE8W6it)|Gl_##US6)lM)x0#w6-7C47&VHun$#)P=lG1}j2QG(FX8foSXM8T= zy!=)5@?2H47YN_LX9wzPmcOc9re@DV6|2(`;o zJy&&?sM$4os;HT!W{nlUUQHx3LichITQ_%POVf^)sJV8t#@Sh}TB}=crk8{co)C5+ zCIcpgR9-STC;l7j;Fky7I&yKj7U$w>+0aT$%Cw9WK7r3s?yWslPL{i>v|RV9>hl1A zvnLclG>;U%(&s3&JWuu1I>Z*2t?3oRV4`HNg>tNlms@k5a*d&jD+_^pYdgBjU6oqy zHRUSTxh}3O13J&mZg#y--nurEYqgu5pJ_kMldoFqt#F@QpS#=|-KwdSeOl)Ui{;9( zs-C;KTkmprGeyk+>AGr^KjlyQoHH#) znSH0dp9%W;w2wf=rt=D4{9iGYnZb4#;`9}DWM~k(b?TOOfSU#1^5LA_@IVU72^e9i z*f;@zny4Y6GDsmbBqAb`CWUlx0udsk z;sy~Atwm%?NJ!FxRF*Mk$e@X%NVxVf`GYoy(!nYji=uzy#-gY}Y_@n@WM`4F87do; zcw-uVW;esW#R!cR-pP&}J8kyQxH$_J;?Urpg94`L@G<54`nWpm{s+Z%fc0`Llpf|q z-|mnryYErlxsZGgAPXWM=vV8YTePjI2F7r=y#z(HIe%gbpnAm675%gIWH7BFO~$Ig z=HRs)DLOk)f3_wyp>;4QegoHD%S35@d)eZ1WMurgEimRZ(*`zNoUv_ip}WI|T=aUv z1K0&I3NJHAgHQGNPNW}M~+I3sbOj#=tL gh~FE-Fd&gBQoe>N4FCWD00000D77#BAOHaX0E%@zivR!s diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc deleted file mode 100644 index ea27ffe39697f72f1a8b0fedd56fd778f545abcd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}E^tdBy<%6te@s diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc deleted file mode 100644 index 7260d917927f9ae1455513e434b4bd232f4c010a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}ETg=&=U?6Pp98 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/README.txt b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/README.txt deleted file mode 100644 index 04dc59c56a..0000000000 --- a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.124-13536b531342 - Created at 2023/11/27 16:02:25 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 4bef45344762b3c64e1c64ed6bcc95b40cba5c29..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}BgrEHfDZ5dZ?5 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz deleted file mode 100644 index 9c950612b07efd241dff9fc88ca211de09c37a33..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 289 zcmV++0p9)}iwFP!000000F{wVPr^VDhX2c4sfl7BrZ=ngWMV>%2MHm|&VVcJw%Og9 z)TaO4843l#i#=`Mk7qxo1*-~z7+xHbBb>cGW)9N)kx1-}(nNX6f~$3Cy(+4lmIi%1O7ERh&76H>m1`DJU7-r}m$R=0CdLP~lTE(% zn_S|11nb6bj_8cSpq)Z{ZHHXd>ovKv>piE*&G;@kS`p|EljBJw$xFRu$gj{!#LRqi zU^iCVvVi(e-+ZsGBIpnVPt0aa5pe8H#RmhWOPk0w$j%xPAx3bO-U?CEKV&c z)-OpdE{RXfONmcO&CQPwGB7hWiZ944NX^O2OO4knOHR{EO0QxLcMWD>VeoAAn%ml- TGfh;57ibFzfNf%6WMBXQjCmrh diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/.index.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/.index.crc deleted file mode 100644 index fb65f202b7365f1ec2c4dee8517cc4c063161e7f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}Bi4v_ljC5w8Nf diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/.metadata.json.gz.crc deleted file mode 100644 index 4eb1dd4f7a5598fce8f9d708d58fd7ca59672cc9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}DHvTU`nO5|RTz diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/index b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/index deleted file mode 100644 index 51f5b8d2767b5254f9395fb4341922a95fa01caf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 111 zcmeZZU|_HZVvVi(e--TSGBU6*Gcd5QBxe*E9uH(l1!ORkx$iTqJ2s8`DC;+X888!d_ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/metadata.json.gz deleted file mode 100644 index caa522e8089ac6d2d2dba6e427e15c1bc315afa2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 184 zcmV;p07w5HiwFP!0000009B5`3c@fDME_+^3ON)lwdN**9uyQ6FXAD_Z4(TMBwH|) z{<~XmUY3QKc{4pU*5MUnvcZF_s-mm~Jfia2Sl@1xrVv4v#j+A$(6)v^&IG(LU-BD4 z!kJ>HsdqtlN$$rx*`C&BUQdN`4jm@Bic#eo8<}_?K!EZ-kRq=9hm4xvVNTpA!1^bC mj*1K$em|=*1UfWLLY}eUNs5^I=mgyHPKd$e`^?a$D*`XAr@jRS=O0050#rtts( diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index f64c31a75f11b114800335a2fba2d8a851e920fb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}8wO{A!cFqv16GBg_Ss diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/metadata.json.gz deleted file mode 100644 index f36ca2920bc91a3f97d36432ecf3c18b781db1ea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 629 zcmV-*0*d_~iwFP!000000NqwiZ`wc*{V#iJRXg!k`s{_majw+)p-q+G%Q*>7fN-kW{a?vZf%48$XMJcJhrXFqS|I|wYIk<}9%i1$e1 zB-p3;PU9G2NXZgIK@vis5{Y-LDl85=2*-5*vp_)@`7-fo3Uev;fjs8rybcs3T(XVA z7^mW(?UIy9mT*MwxU{WID|;g_5`rEctJI|yu~>pgy^{cNDHzm&cE9NluPCgId~b3a zCsaBE?dG9Er;G9_TV_hfpHfFh!M*zbG@XsNw)?pzSTw^##y1VQImjBTc-5IDqnc2Q zPfcfx^-D7AeH4M{&HjGko0jW0A&mrE7Kg1AA89`!gHF;~fphj36P)4Dc21Z5N@bWY zy>J~01Nz)wnW-O&V9ISh4R;lmn@>ReK;;(ktE;bhx?ixKY~J)3Ds6r~_*CY19;(sFgHM|R$Sjg*#^~Y~#)#mS6X4-1>s*^d({x;?TWWzh)4USnLA(#q+&MMW4mCDWO%#$THB?HW PdS%%^5b1+mCkX%m+6yPH diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/.part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/.part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.crc deleted file mode 100644 index 70c55e062cbce976f1a9a9501187c85711c4d40d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}D%}oh}Lh5#$1< diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c deleted file mode 100644 index 18276160798b0327336a0d846bcd7e9a0ce8bc8f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 204 zcmdnaz`*bWh&8tA|5f-A%F1BE!N|apoKa+WJdlaW(V5Yafti6pgu#Y|L4t)Lnc;yw z7ti5iyM>q-Lm1sP6rb2D@O-dW5o!E*L7R!uJz0O@-j;7dx9p{P7?NwR@h&@&e%D_0 zgS|%E_KVt#(qVzYK_NwndC3{6!F~ajmU@<^CHaO126_et%(_-JQi(g43m&tV(^b diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/.README.txt.crc deleted file mode 100644 index 46b5687838a74b08a409ed7002165d192895e847..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}EUF$=Ct_60rjz diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/.metadata.json.gz.crc deleted file mode 100644 index 63190d7e43b8f26b9be44ce6688ad6f7d0cf61f3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}7-$=y(AD5iA2L diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/README.txt b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/README.txt deleted file mode 100644 index cd3d982528..0000000000 --- a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.133-4c60fddb171a - Created at 2025/01/02 12:26:15 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 4bef45344762b3c64e1c64ed6bcc95b40cba5c29..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}BgrEHfDZ5dZ?5 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/metadata.json.gz deleted file mode 100644 index 9c950612b07efd241dff9fc88ca211de09c37a33..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 289 zcmV++0p9)}iwFP!000000F{wVPr^VDhX2c4sfl7BrZ=ngWMV>%2MHm|&VVcJw%Og9 z)TaO4843l#i#=`Mk7qxo1*-~z7+xHbBb>cGW)9N)kx1-}(nNX6f~$3Cy(+4lmIi%1O7ERh&76H>m1`DJU7-r}m$R=0CdLP~lTE(% zn_S|11nb6bj_8cSpq)Z{ZHHXd>ovKv>piE*&G;@kS`p|EljBJw$xFRu$gj{!#LRqi zU^iCVvVi(e-+ZsGBIpnVPt0aa5pe8H#RmhWOPk0w$j%xPAx3bO-U?CEKV&c z)-OpdE{RXfONmcO&CQPwGB7hWiZ944NX^O2OO4knOHR{EO0QxLcMoP@VeoAAn%ml- TGfh;57ibFzfNf%6WMBXQjJYDO diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/.index.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/.index.crc deleted file mode 100644 index fb65f202b7365f1ec2c4dee8517cc4c063161e7f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}Bi4v_ljC5w8Nf diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/.metadata.json.gz.crc deleted file mode 100644 index 4eb1dd4f7a5598fce8f9d708d58fd7ca59672cc9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}DHvTU`nO5|RTz diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/index b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/index deleted file mode 100644 index 51f5b8d2767b5254f9395fb4341922a95fa01caf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 111 zcmeZZU|_HZVvVi(e--TSGBU6*Gcd5QBxe*E9uH(l1!ORkx$iTqJ2s8`DC;+X888!d_ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/metadata.json.gz deleted file mode 100644 index caa522e8089ac6d2d2dba6e427e15c1bc315afa2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 184 zcmV;p07w5HiwFP!0000009B5`3c@fDME_+^3ON)lwdN**9uyQ6FXAD_Z4(TMBwH|) z{<~XmUY3QKc{4pU*5MUnvcZF_s-mm~Jfia2Sl@1xrVv4v#j+A$(6)v^&IG(LU-BD4 z!kJ>HsdqtlN$$rx*`C&BUQdN`4jm@Bic#eo8<}_?K!EZ-kRq=9hm4xvVNTpA!1^bC mj*1K$em|=*1UfWLLY}eUNs5^I=mh*;Qx1;xQ-7G!3aKOlTf5%Fw`dS4pA%DR53_EK+|q$_U=?kmTR1__ZnI<=OS z8_b2;r%_>Y=E2bw_)FHpwT_;%5Pd^QH&XM$MNCSN;L^zIQ|E6g^rL=0%$La_o;-Hk z-G(>_k2k*A5EL%{lC^UFN#B%@2hsF}zDmvfXc)~FZ8l%i1pOfm0k6;X(P)JhM`%UP zH*c1T9qWlKzh+5?tPJpGsFz80e85zn{@>TY*kqXX!s92*xGvf=w1P!fN}9^Otm8Ms xwDwqS0C~7ET*9{Jmdb_mLUAw4cPJqz6lVt^psnq1eukzs`~oCMz-fg7001N_s~!LV diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index ebb4a5208f14f44ba59c8d668fbcbd2d3b7faede..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}88gY1on_)R+MPAe96? diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/metadata.json.gz deleted file mode 100644 index affb24f42675838fede8ae69229c9b0d286e1e1b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 628 zcmV-)0*n10iwFP!000000NqwiZ`wc*{V#iJRXYV^zI?-kAf%{D0rk)bp~X9fZJae) zFI6b=f6uJHZKxC?PejmXAP$)c1~0(ozit@Se5)}o zs5~0`Fye}b0+E?e9V<4n-Uy7Oq{qiRb!GTGP#`1cEC51D4n?5aZ~D_o3mYTXncM{- zRXy9deaO-2GJA?wvDWc7(@|4!um8VH=i^;x@U4D| zL1&EhGcucl?}Kd2{(j=Nt$|yH4C3v0z$z&&(tbh)wWN&(7yKV4^t#z; zmFrL#(C6mLSpQH2*KTWQnB`bvJ_*4CRXfCQZoVh!8V!K@iN)UJyXv0%Tt;fm*-L6F zoG;qDW~ee3BhCkUHa0@vk3bbns1Tu|+3HG<__F^>DHS0#J81#enzdkc#;L*PlNb;` zOhJPm=$rQ$D(NRdT((f+*foCtux@Pjxe-(3!%dgAk>G`rT_$y<4~q%Zymzj=sk^EA zdZ%8J@*g3e(Y!dccb-Ve49e-$tHeYX-&3jXxIXCd?AI@5%e1Y5Z4IeyLf-<@?%K?x zEzdUl8*e}vbXnhL$-UsuoQf^D;XyFH{zM{$6|}JQ7J(0`hq0@-lWnAvdzd)wmfab~ z<|P5j3s< diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/parts/.part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/parts/.part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.crc deleted file mode 100644 index ff3efb74115b772fcecd4ef72b1a35c05f868e4c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}D()?%Oc{73c(@ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/parts/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/parts/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd deleted file mode 100644 index e03622879d36a95fdee5a706a94fdcb5b53b3eb1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 208 zcmdnZz`*bWh&8tA|5f-=%F58f!N|apoKa+WJdlaW(V5Yafti6pgu#Y|A(=sf=Yc(! z$l+tVg_sya7~M4#pV%v~e6Uw({CGi|iP1e-f8pMiZ$h{1rFj^VYp?MxJCT0ZUKOZ7 zgQso#MQujuu)yG;kfOxAasx*T!CvQojq;yfqIzx0X(g`mnX|Y>#wBEm!ZM$B%^&0fmEx$(L z=>9Q0r6}Q_Qi(H4s0zMAn+wv)gR%TV!el&tnEaG|7%L9af9kMjVA-{@@gELRGHqX^ z0)sD?_~33x{0rXBSiXUHq)$ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/parts/part-0 deleted file mode 100644 index 070cc6a220a862ea72da8451d0515eb613a6e41e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5585 zcmV;?6)x($6#xLin*abPwJ-f(!J2ig0J;Z7FGJ9ZYMKWCzQZFt6kj}mb@(9u8|nXV zB&e-9imbLqOLE#gF_-1Jis1a0bxAQv5mF?{=WYfCe~nNvSQO7G1C)G+1?vLd0@?yf zg*+N;%wyPHW{qmjMN!ORopLuS>Q1IH%T&PS+;_5ObZX&pW?+DLICr6a()x`~vG}H% zjd4mUR;tC^ySzI9W^MgR!8C*#k(8ZqR83W6m~ehu?Ve0Wg42=q|4{ zxclFGP9aQFWOr=OX+?z=d^hvwbfW62k?W4mX@y*;jrA0rQ;JA4)pyJ}t#BqCF;G>0 zG3WGwZLGAGb1GrURV#OSu=AQzi(D^o?4Gz9Ix@M(jiu@)auPbNU=li|h%}a}vu%VO z>k>M3=m=CH?Cj~F(~6Bcai%K0(?Aa!bV~6WbV^}Dl+S5}Ms=pwQFC?kD?XP(SndwWhToc+{E^XF7T!C1+K z%7B1S3bKJJ3LZ?tK73=&w1nX?fKZhq5C7Dx@02b03L!ZD{8M!?9NIu@Lgzt6O*o!tEGF{cuINxsSWnMBJw$DBs+$9`jppm&Wqg}{E& zkrUa^`1vOBJb02K&9B9rPFPDKojH9VC99E{Oe#^Tc;8f(S*PGzPu=z0avK~W;gk|G zjbR{*@zuyEuS=@vs#BMlPC;qiR`;Hi?8Rjxo2%Yi871vg)O^w@Cc>uVT^2Xl+jt@u zL}wS6{FUInuf6)gw${50enWux1^KGhL9>(Z&f@|!x1~(&` zJ_7>cp}0Ez#pOxJl#Ow!*s4>@E>p>cS!XW;wvic`9qk-h$0TF^ z)7{Tg?LXj?Zei`o##mLSOSPVJieW6HrgL?9t4aGpk%;Ru5Q(@rM^lbKO*{6TIeQ`( zj0hp(exCE~D{V~9geU)tuFeGo7ZgyS9sAf*M4BSe-yHpUQYfW$^ndkpw(dN3I9Rb0|Ou$UhhBkH=P8gS(}K(P;#Xm0Y+AnQYN6(JhK`ilW$@ zPQB-tip-r)QJ~~8EZyIDZ{*uNlhLU})b(cY85kfwLx&1}V@vnJ62|L0UuZSV_3p?NMF(yMrqTZvW%U|D$uF6Fs9;U-eYU2 zt0VUA2s}@O`8T7}397}KslP1CKh<9SfD*=c*J8}FD5gNEWR*_E3awJE(%hUFyO}(T z(WykeI?{ODM2|ZqtBhOjmQzb*Qo*Y@WtG^}oNGP!5YRB1T(P6!J3ul#1d@P}l+me#ITv4+r7lakOg(rN)uQ4QbdOO``kQaL1Ax&r zrrKP<=+9Nw8hlD)6l{y47RxxrFjo{~;{xYX2q2+T2#l3nFjn%@zSE=G2%}xT)>G8- zDb<{dQ+6&(wS_5{adSRclI!o?yvyix!ZcM#N6bH885xj=a#wQ?TKJk&vaYF^2%UP) zUBS01i*0VW*qN#n;Rjnb*+M}E1_+CFG;LhLH@Ox-N1RDxu(iX3af??)PUYRHoGVqm z*qoYcGU_eHuw}|9-v+ijiMY@~8e6PomE?=;TUvUq3&N-QX7L#D;RWo!QO_==hUJx$!~YR zA-w;&P<w_ar{hG1#&O~|Xo>L1fLXLhfxhj#C z+`SP~LIX*x0u-rAzDK`NdrmLxC|WAy>eyfGIeic{)ndm+ zo>L0v&nbjBol^?XeHb{M(}{ZDbWSbUbO8=BAk3Mr9VF;Z%CS?`b~DI)xm!?WUaF1# z2P0b_qUz5Po~n+N`aF?ckge79opyy@z8GOcPY>UP#U7->n~N6AEugI?($eFqovQrO zSp3lei5YSX?!|sW7OPiB8XHSw+CPba51uzBHQ!c*_-^KdbS|Tv098| z9Ad=?D?%PXazqUuTN`RmjOk8stN>niU<3t`<8DcADzc z4X(ydeUA_z5!TaB2%(5iq^BI&-%phapfKUdPdER5>PHA64#-+;iXJR|#vl+vHlC+m zz1qS}^s7lk_=)UhkSzu)K=9spYOa49?N9)ax6Xb!H$a#}9E*Kq%NP6UYdL$kqC0!} zVS+AlWVsqx+RuO`c|ar}KAky;W4W)2_m1dy=YaNRdlATH046VV^Ie{`SmBubt6OXm zk=fsjB_mA7@uxB2Gr z0M0sASLiGAxYpl|`^%lMzD~xUtHYSBTKa9Vjzl?f9_L@v0TLBbC>?rWYk6CLpStl& zMu3Yg2Gu+Lk~O65OAG|?7W1}1xv}_R0V>u~u(tay&mx2%x-Z6@NgnUP08Aq9j{pyz zYH`MoebzYaJ9%%==>wBN3={S)i#hw&;R4&39t>6Hb;SI&RoWUln*5pxkTB)H8j$;3&i|htC&V?4Pvo)_&iKLqLSZ^6xug(gMc>T`b9?$@qDyeLVRY zpVLPM28cI+1s^g7=8_K*H{<6@SbFxL)5l(N-3@EbDTU>(GQa?XlCDH8H)92g?Bf}R zQXjb2`iwQU^U{?I#!4=7v^6Z&`Bne|*9S{}*~?tc>BMiy*bsyHsKWY?V@#qM84-~r zNzwsA3K9e*fs&L|LLU+!@lX)uI0m8^#u#G=A%qY^2q8p3ghoIl5fPCO!&-4|T1=6f zj>eOZjIYcxS^2J15v!IdH2jcnIx9w|p2Q2hN2C5HhY;!xl+7itcdI&c9QIr(^MLCE zLXBl)_xGl1CE^64tD(AT*|_g<7%#wWzFc3`;kUkUvEUHWxq~fq-cE z8R$QOyz=BCN*ARJ1?vDpB0CbtHb7he%dbEVe`YWD0|LZ8gDl1R1RH3rMG0_2Mp$y% zs!tG7^*K_kt%$ujEi8iTC3o0etXyXHP^2V5DLIz|BpS60~aiWKN&U+%xT{m8aRS%b|FTas{AEeN2h;wJ&*Z+|SPF+e$ASoHyB07&T6 z)T+5xt~s5=>5&m=;xX*XamP5oxvL9(!mu3X$3&0cOr3k!crhI(RkYaYoU~@Hb*5dA z9;WHzHqVCG6Jmh?1e#qeIS%M^ctAxcg-_4sD43r|5EtOMu+AtP(f?Sl=E~9`>jL|O zNT1I5vTi%;x$Gpm-~*dO&xWp>6_*VMd}ISJ3I#iEJYYWi`_2Pl3XOBmQau<($M~Gr z=fjo?NX0CBmjh*|*$0r!jR3!-qsnB0p5y?(PUuWYb5wwym$R(gah{Cu3D|iK!C7)B zOjO&&DsZz&4z9bd3KjF?z}gW(q`n`eR2F=59}q8JnwGTa+*Fxb&iud~(;9hzWp>V* z_|>(v+-{r0My0X}Y-sK-}fG$QA%W*8gLJ>q>$7YSV9Ik0Z>etXPQW zanNPqKOdZodNSQ$&e^KLIJ~Xe3Me6z&7V!SiS)@1TmG}s#YPTO6}n+AGV{#H#ghqo zjH}F^ncWNWa;>o$;$)nR(2pD7=wsHhKFv0^vT=qyb16R%FTMwnOhF4j3&bpUJ;i55 ze3leK*DCa4%0k+q9TXW_Uhq%G&8zQ_(o(dIN3EEmbiOHs((~n0oUMTxpD#R{8K_8S z4Js=MZVL1-q#UfGfKEV)@Db7Une7ZBqtuS03zmQow1PwN48eUwK+B<^^GOMd_MR(m z)P_2tmCYPR(l^d^ns)BDt6C&x&0FHv6)uV1ppkbiN^_(Hp05lBgP?$gkwe<(Wei+6 zaZhXVF32O?h>F;wxFBx>aatT`SCJ&BS=!+dPOA~~Kxy{ zZ^Sr4J&>&}Ix1DqX#SqKiy;AW8Z<&7A9Xi4<^)T+(NKZ|89)@oATMyTSoMO^-}8vm z47lT>h(3wWy{}62&KBW-le*PpoKIZ=_}>Q|=nSt&{(K)rexq#x}6G`K4z z;gF)jsv5a5QxSy+vx$Cm2WM~|H7ukPLL$Y!kJ$K&wvM}{c48Im91#i<`*ql1E~zJO z*7Zw9Oa`Vo$?|fP(z|75JnDF1Jdb$xWB}kzI^GP$dY+d z%#Y;fCmThmS^Xx}3@DV{&bA_l%_|FzViG{(lm}Hvq5;TvzA=d>FkyzYKT8>+Ji_`| z(hZ3lj8SeaB%V-yq&TF@Q#wt%utKDJDe^zF{BGJOv+j@D~)67--S2WU^ z4_aRlQ;>_|8s1(M&nJH4)^c1zN_mo)OE@x}!(|k$Qle(iI;C6#A`Y9#&8w$#ER9qJ zdU_Wp3^6{W8x09<2o3=WG!@PX?R8ju zcGIlzmrDQiKt?gj2W;^7=o^Zw*bEC#QMCuw)*O(;F!?1?Xp)sN>m(!*O)PlyGRC`J z36^|)l(7{$gXGA7$onR-B;kZpj7A9gJiG}Ji{TG(@(D*`XbbT=D2^3mh;;id6eLEC zeOT%oRyp%f+O2`y$jkjUIT)*N8ssfNc3p8!6>ET0?fO*@HmCQYx4-x?8^d6&wR3?+ z0)yk-X-Mi2Ko1W~{|BRv{pW(G5)LxR#wO8;hJa+N zjgPilq)^F+D%|=Mt!P+Faj^ILDCs^{+%0PsvPdbNQNSY_X(8{ps~94h1Qb`T&1WVD fY@o{Mrf>6DEbamg000000000ewJ-f300961FK(E+ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/.index.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/.index.crc deleted file mode 100644 index 7013243ff9fd1b59dcb832d14eddcd98d1c6b955..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}Bhf^4Jdm6r%)F diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/.metadata.json.gz.crc deleted file mode 100644 index 359650e81674052c8db801f7ef6696b142f31b71..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}AVWt7<<06qN(> diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/index b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/index deleted file mode 100644 index 2196f12697914ab3e1fb7705a6b30e48c677e29d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 111 zcmZ=~U|_HVVvVi(e--SmGBSK&VqjolNzN!TT;$Kh?C9*w=*Vz9kcr6=$dGOPctM+q s(VfxJzHR$OAS;B?J%*Wu$$#OD_A>?wKx2eqMhVYkU|?hf8ii^S0NR=wC;$Ke diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/metadata.json.gz deleted file mode 100644 index 521ca22d19aaad9700d820d725fb93cc43d0d61f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 184 zcmV;p07w5HiwFP!0000009B5`3c@fDME_+^3ON+5wdN**9uyQ6FXADs+a?$iNw#1p z{dc$CybJ>~`(}G+jKv#OntTA6XRAVhM$>8nITP^0bWLvr z2^R`oO}+JX7v*8p$?m+-^KvSbwP-QQbqFfmxyZ!F00NYAo)q!qKV;0@9#iB+0meP? mb5f+a@bjDo@9Eey5&4bff=R^Gsw3o+ei**=>PSz|0002eL{k6& diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/metadata.json.gz deleted file mode 100644 index 50f4a7346adcfd7e8d0f7dfb335c2ad21ecf6d74..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 381 zcmV-@0fPP?iwFP!000000F9DSOT#b}h5t*Q6yjJn(ULdWKtvy88@`lMvh;3?rb$R{ z9cA>tn|5i(#0Ov6=G^a`+}`9wc2Ys{4bDrW343^)&r(VbT&jymrjywunWc9CyiAkn zlqVt~lsMRd11-Q84pd<|9s+z(1A=@T;<2$1Dw}R`+AFi=imlPr0#DYNPY_t_cqNqt z&z4ePwyCXxEFeG_q@C+Bz*#XP6AEjb3m8Ze7tf-B_#Su_sW`&yP!Yx7a{yt1)3 z&(7I@ZJW3B3mTRwBWh2>2B5&_)S2V(3w?#Xb71 zRCD~luZM%la)%PbeJO;gV-GRleJCyZ%I~m&&+hXs#~BNOhpU89jGk+%lpduPsG3)3 bpd>6z6WTVUMSp+udTaXw!e6ed@&W(=cx1Zy diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index b87073cd48685faf47474df6d566664ca194d234..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}9)d(oL=m`Tht1ASVSE diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/metadata.json.gz deleted file mode 100644 index a63aaa6698f0300b757e83fa52c499ca5e2ee4f7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 621 zcmV-z0+Rh7iwFP!000000Nqy6ZrU&u{TDxNTGAyU1e&+PScOzg6I6Rxg(4H*fZO0i zj+x3({(aX@2_#`ssnnP90OZ``d(W|t6z_?k8z3Q>3I@->X1~78bBH}+k?09_Bzz!g z7HBzuybG%YGM%nWY!5L#seH?Ukke zpb4(?*YYquU?z?d(w|hV5x=_nYUmnGfhHz&uS()+AQCPM6>;`l9dqDY82CoSj$EvS z@0jA!%95l2RV$%FWSEs|^~sE~Je$O$l@lY4fN2Al#ngl{p!ctfPSzYdqBNj0DfW*V$ZStYu~q*12(jsfy=z5 z>(HR*n47}mjEWVEvaMix{hr`EP%3S#x>w+zD#JK6UP~9{95PHo*Kz%zJYGF;{>^xd zlpBI)PC&3%HoA8Et{V(nz3!27YdUcer(1^UHrmiU)YwE1(MYN7P^F}vRhIt%rfX%l HItc&(-oPjy diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/parts/.part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/parts/.part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.crc deleted file mode 100644 index 2a7dde15ed6ca272aa3baaef8081bf56e1baa88e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}E?);n-II6`=%z diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/parts/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630 b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/parts/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630 deleted file mode 100644 index 1d7128c9f7ba35bd412330fdb4772c2bb6278c44..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 123 zcmYdeU|^^MVvVi(e-*0EGBIpnW@KPV&L}cm~Px$iT$N zz{J4F*!b~+HWQ;eBV*h4i$F#Qh{?#n&EPD-aENhLKzVauvTJO^BxjAw_OqE(nNFON P;04+O0$`hf#xVc@Zp#}b diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/.README.txt.crc deleted file mode 100644 index b214441aa7325462494ffb7d5ff162036902d937..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}CuLd+`GR6jcNu diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc deleted file mode 100644 index 2cdf8bc33c948df56958c0aa56c4184bb9c04afe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}AXazvC_d6g~t1 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/README.txt b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/README.txt deleted file mode 100644 index 3a80cf1947..0000000000 --- a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.126-ee77707f4fab - Created at 2024/02/05 17:25:57 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index acf12b18f0fe52a544cc6f65bdbbc7e722339f38..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}88TyV?~15`qH^ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/metadata.json.gz deleted file mode 100644 index a9459e7d89b84db892bdd4fb907f3b19a5a2899a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 324 zcmV-K0lWSmiwFP!000000F_crPlGTR{x7`_6P?*&dJ~5y6O&;)NC@fL7pyCVly(Uc z{=09-V1g%mYM-w@{pg5hB@jwpjQ|5;|GtZB;A;|)>Od?Z4KgS zB2Wb$LS_|(tby1X$8G0o!7Ma%6+Ifm+5}0dnQ7CYVP&l#vuL6t8sw~$s?7=~#nm%e zF6`i(WnMSL)^toeG39_RQu?&_ZBLiBi=Hvq1R<%5dBXvl_ZxCY^`hKP@-Zox35V?u zF3-U%?(`)J@753Dl?n*|luG=PgsS34jG2iJ9*pHT3bvcg{r0czqp{*_{ihE*11p!# z-VZ!P&5V7Pcv}1?#4m1ciT^KOd!`ErE2o90I?`AqF&0gr#W0bYD<|;T&J?}^OjzVK W9_f7=v(_};)Bga@3U%hL0ssIragzlA diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc deleted file mode 100644 index db43a2f58b1e4167a3a160da7555e44f2865bc2b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}7j)m#_)|62t>% diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/part-0 deleted file mode 100644 index f37af194790dd5fab48b2efdec1f7a7336c5b8d2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 158 zcmeBVU|?7V#2Q=m|0*nd%FMvR#>nWJUTmeWUz}Q4q??jhl31KtQmkK+T3iyJn3oct zlA4twfE~lY$iM&q D%R(*+ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/.index.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/.index.crc deleted file mode 100644 index 3e56650a624d2267be143fcc3b6e7eb7d7ba26fc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}8{_n-K&659|VG diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/.metadata.json.gz.crc deleted file mode 100644 index baf597cb48f756b1bbb87d189b0cfb9730a0253f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}D&z%yS0-5xoNb diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/index b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/index deleted file mode 100644 index 88a0daa88afaf57588aed5034394c25769561a7f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 113 zcmeZdU|?_tVvVi(e-)gcGBSuTGcd3)uq0;`86FR0Vsdn5bYwVuY_|{-V+f>7*! diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/metadata.json.gz deleted file mode 100644 index 18bd4926437171de4e3e3b9af71fc03184104620..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 184 zcmV;p07w5HiwFP!0000009B5`3c@fDME_+^3ON)lZOu&tJt!zDUc^IMw@ol4l5D|H z`tNSNc^L*~_RS8^8jCj!(Rc^4D6+f~PzU82ZCtli4fy~v&sK#1ji%ECav|V_>6+dM z5-t?_h6d}KKFY(Sv)y^4m(^S-Ytdqq>kw4BbCHRU5dpY}r9bH8(Xb1yYXg@ouAv=J&NZ0=9J1tn)LkILw}%R2dU!HP%b4P9#X9? zeJ@>weYW3+^)@(97mq!6|E7=@r!(r>^#M8F2g6`%^VeVOJ-1VjE??Q3RNT&IbhUBX zdd~uUtie7T99*ojLgg4{2&>6;X5BI8Sv$+^_bTX-T>){oj$sEq-=L~4|Mzt;nhLJ1 zu(fC>`AaCp!(nCAT5 QZ`4_vKWDWll)nN103c1R=l}o! diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index f7f37f8cbc1bdb838be387ea93c566f96222adc2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}E?=_hsd$+)f?vhX0GNHZ^o4fk1LcC`Fa3sROi&RS22n1iS_Z zIc6$D`R_Z4n)e9Vn^Gw3X zJH&WT!w^D<$r6{m$Olg)66aKOSRD7@kIMzj0u{oQn~6(fm`ic+M zSW{(ulTplN`ZVcUONh98tg?=r?MH4Tmr;)T)!8!Yj z30A*vTIXfIV(F))SJt63pu2WuqF!i%DH2;A9*S5BrhxE?$}PrkZ@;DKDjEU}QvBY; zWF3KkiCC_f|C$kp8;s~z&@}MH73afvPP8ZpVvwZ{%6TMdUR)cIBo|q%jMQv2Xj1a8 z1-Ddnrm3aoli-J6QclC~NX`3jO6LjSmjOz7UEyy5tXdEI9K$T~!N_Gp1bC6~FH5?T z!qR_(ymQ}e*KN<&b9%|je~f%abaB((`QMc8LA5*0E(uZ1_f*JxrVapjCQvt>Zio8j z!0N7zPM;2}PH%1ZI<{dAJGM12*MVh#-M7;7!tWX78*rlo_f`Fg#1cCwW7+8fFEWI2 zs<#z7$|(>gj^6E=c2XTcaQT+`CMHK`Mn?uF24)5Z5e6F;1_>61 zWQGU!Ts%z7hmY+RVqy$obk|UPVz0pS!CpnA@#6(;CPw#U{e^p5z6ss3m*!zeuD!;) z>_qxqd({v28azx449Q~gA4TIoUbR;SO6J|(mV0~KQ+pYnw(S?S8KuJlgM&hf67!NX zQiJ^hEG_jcO-u3(4Gi=Q3>b8+YNQf(E*CszFAKDtL4YAcnf2b<)JIBXona^Vrc7f` l+N1TPbwbEfhQv7Do1sn@Od?Z4KgS zB2Wb$LS_|(tby1X$8G0o!7Ma%6+Ifm+5}0dnQ7CYVP&l#vuL6t8sw~$s?7=~#nm%e zF6`i(WnMSL)^toeG39_RQu?&_ZBLiBi=Hvq1R<%5dBXvl_ZxCY^`hKP@-Zox35V?u zF3-U%?(`)J@753Dl?n*|luG=PgsS34jG2iJ9*pHT3bvcg{r0czqp{*_{ihE*11p!# z-VZ!P&5V7Pcv}1?#4m1ciT^KOd!`ErE2o90I?`AqF&0gr#W0bYD<|;T&J?}^OjzVK W9_f7=v(_};)Bga@3U%hL0ssIragzlA diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/parts/.part-0.crc deleted file mode 100644 index ec695bda7b936238f5deb256986cf2fbf454eda2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}BiN< diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/parts/part-0 deleted file mode 100644 index adfbcf32cc05d70ab4168ed1833857ff6e6ea88e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 179 zcmbQoz`(E-h&8tA|5e!9%fj%=;P>Sa~S3JIyeZM!8q zxAPn>w>>R(@{) diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/index b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/index deleted file mode 100644 index 55e960e931bb88a950f33a3e8604a4529fbc0713..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 104 zcmcC#U|=u;VvVi(e-(@-G6I>*3@phRMTU#~nV21&of#b&ApGNjOiYeIt|Ekc_}FeC aCdLp(cc1_Rg9ywBk%bHlj386c%m4soJQa2T diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/metadata.json.gz deleted file mode 100644 index 505696c221554258212a8ddf2a204174be3577a1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 185 zcmV;q07m~GiwFP!0000009B5`3c@fDME_+^3ON*8V$DqiJt!zDUc^IMw@ol4l5D|J z`tNSNd07T#=Isp7Scf-^$p#Nf%B&~_R8jldSl@1SO(B9RvXvBI(6)v^Ed;!)O18L%y|B%V^d+ZaaT!8gY n{G2oy4*WE`F$6j`O+ua_mrNq2RvjT%hH?A?EnNng&j0`b1YS+q diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/metadata.json.gz deleted file mode 100644 index 9af33596ce4d961c44e1554796a05f1393749df9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 370 zcmV-&0ge72iwFP!000000F9DSZ-Oush5t*Rwxlxxi!*(5n3-jJaBB8q2&v_&%?c%L znIVM#z6$~dpY}r9bH8(Xb1xM+N(J#R7%O!~*z6_n=aiheR5y`KC;r6q=T87UnS0YI z_k>3%F>nL}nE)FbP;t)P8R46Xh~o#0cV!M(*)^Byr1XI+wsocut}60`JFsjEZjX}8 zP7$>{%{v#nYV^ocsml0N$i%XRF}xh;u5>k7wv?hzq9ZV>=?|1_ue2yEGkool)0aACTdFFbuXPfAz)QGZ*-D@ygzm=5F>(m+MYj z?O2G9)!1i^gA+Qw~)*dsSHnZG(FT)<$<{;X23_Ixg8dZJyzpsVSlyl>x z!{x_97hMlsz`8GmZRI|!=l4O>vhX0GNHZ^n&34!JgC`Fa3sROi&RR|g91iS_Z zIc6$D{_i_Z$R|uHm3mVyAUQrC@9R8(4@A%n5T87V0A3-O{JfiPfwhT67B8?T{xgYU zwoCAyMiE4ikU1`~H~?l6iGRpD%=TLdhQ$JAfeGQt)o4Hym@2tr>X5ecs}U9p#aBLN zaY5zY+Qtc2JQf&v5Xxz5>p0pU^`)fy=R9+6g*;Lq&n?gmkM;`?N^&f`Iws^FmFeRj zem2JH5B$-66jSB7*4rcDpwymn_28JPIl)12pl1dTL2a6P)tD znBa9fj(1x2D^WpOdSM+p1G;HeYV$%9+>qGt@Q}w+FeOA!RIM?7ef=#>H_-@anBw;) zCaVa9TqbJ4gV&5W++ak%grY06^=_SR%kkVLqrhnBcGL61?+Fzv7{ogfnEC@r6m~GiveN}#WC-I_ zww5-^$q^>LePg?xE)O6$zdsyeN}u2}Cm@11HgbC1w(Hur^x>Idv b9*Wq=3^5UH>`iFVvVi(e-)B_nHWNt85vlTGl~os`7<#)Iy*BuGB5%m10Ta1dk&uCflN$} z0w3(zL=GR@EyTna!swp-z+Q%jft|sCMerEo%mX`|MSVkrq72x0fu@20*kq6i0NGp? A_y7O^ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/.README.txt.crc deleted file mode 100644 index e0d9c28d3dd06cdc7cf8e70593affe2dc407e990..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9LbfrkMA5-Od?Z4KgS zB2Wb$LS_|(tby1X$8G0o!7Ma%6+Ifm+5}0dnQ7CYVP&l#vuL6t8sw~$s?7=~#nm%e zF6`i(WnMSL)^toeG39_RQu?&_ZBLiBi=Hvq1R<%5dBXvl_ZxCY^`hKP@-Zox35V?u zF3-U%?(`)J@753Dl?n*|luG=PgsS34jG2iJ9*pHT3bvcg{r0czqp{*_{ihE*11p!# z-VZ!P&5V7Pcv}1?#4m1ciT^KOd!`ErE2o90I?`AqF&0gr#W0bYD<|;T&J?}^OjzVK W9_f7=v(_};)Bga@3U%hL0ssIragzlA diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/parts/.part-0.crc deleted file mode 100644 index 19fddf78f6d68308a22d7ace2105b912208dabb5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}8Aktdb7^66^yU diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/parts/part-0 deleted file mode 100644 index e5f383002cd5f6aeadc23fda76c3fff4d3539a3a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 179 zcmbQoz`(E-h&8tA|5e!9%fj%GB!O+=jO*v!AUp~Eh?GNMC$s3jB zDl86^NG+3^FPEO3*lsWU=~qbHj}(jbFSmd4x->~ns@Y?a+T%$k8S8TdW1rZ+aeH6g e>6OIV!^*_aeDiNcb}Ab$&~+dHb|C{J0|NlBHA)Wv diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/.index.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/.index.crc deleted file mode 100644 index de6e63253254b0980d05f26595bee533acb3eb04..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}8AH8L$xm5$pp6 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/.metadata.json.gz.crc deleted file mode 100644 index ebe47d531cb4f90ce477cac8b49a6ac9a62a0e59..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9+AzRDN?6F37> diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/index b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/index deleted file mode 100644 index 55e960e931bb88a950f33a3e8604a4529fbc0713..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 104 zcmcC#U|=u;VvVi(e-(@-G6I>*3@phRMTU#~nV21&of#b&ApGNjOiYeIt|Ekc_}FeC aCdLp(cc1_Rg9ywBk%bHlj386c%m4soJQa2T diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/metadata.json.gz deleted file mode 100644 index 505696c221554258212a8ddf2a204174be3577a1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 185 zcmV;q07m~GiwFP!0000009B5`3c@fDME_+^3ON*8V$DqiJt!zDUc^IMw@ol4l5D|J z`tNSNd07T#=Isp7Scf-^$p#Nf%B&~_R8jldSl@1SO(B9RvXvBI(6)v^Ed;!)O18L%y|B%V^d+ZaaT!8gY n{G2oy4*WE`F$6j`O+ua_mrNq2RvjT%hH?A?EnNng&j0`b1YS+q diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/metadata.json.gz deleted file mode 100644 index 50f63dd339ab46dacf981a77d13cf310dfa0c0fa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 370 zcmV-&0ge72iwFP!000000F9DSZ-Oush5t*Rwxlx~7%_cwn3-jJaBB8q2&r(+5oRSk4@+y+&i8p!iW={Y?5IhIj zG~)+CiG~a`6bYDEgN#dVPY7RCL@YmGysb;X%C5UKN2v~6vaQusVwgH%y|BJV7q9G1DsE>}x?DSL zwPPVZ)?%M64lYr7sR|4;gf-+cv;J7{yqo3bdl?SMwuGc#$GC%mZ&6j}|ND9vO(oY> zSX_RbsLJ)=0@gz*>?`+S1Hbp9E=Q{pkcXRuQFNYLDiy|xBC)dmfErT5aCp$RnCATb QZ`51cKUnFyioXH?0F7L*_y7O^ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 5679cb0be4a267a2c7557c73f43f62e77c9b1831..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}BK{cgx}0bG--vCu0S( diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/metadata.json.gz deleted file mode 100644 index df806dda7b95ce6081029b9ae159febefa7b2f32..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 642 zcmV-|0)71-iwFP!000000Ns{dZ`v>vhX0GNHZ^o41W0p7C`Cxs)B)PXDuhgY0$ziI z95a=n{P!Iv9O1&u;kQ|?n_jMk?MX9e2W-l`2q*xK9ex= z_c7knFoY0dvcM%jV&LmU;vVx3^TQ6&>>v88F@5@nUZ^79n*GhHR4A?@wJOt zTu^y1c2Ud~j|4_0LUpXfIE{8k11ag@InP`e0S^_(OAB<%!^0AUk{k7AGTiWN&sudG97K)21xM8D7k*CaMPO!8O?riAc`stv|(Z@;DKIvN5EQvBY; zWEFvc%UCTrd(DW$4My}!XlnT4iu++aCq@(mF{nZZ6(UmfR9qR6Bo|pMjZ|#ZXwvep z1-DRjrm3Okli-J6QbEJ-NX`3jO6N%smjOz7UE*&6EE^B|9K$T~!N_GpB>0hJmnB_E zVd1|<-ns9l>!#=HIlW}%KSn+yy0~fY{BKJ4pxm8$mxSo%J1W%!*9QPR6X=^(yJb1F zXLlHDIk0NC?5^2vIepV<_2AA~*@12KZI+%Feov`bgBR@u)AdIZE9{_+Wv2_g$PmV< z{zjT8Cr6mLX4|xTNqGRl`GesQQw9W|IRU}m*r?<5E!#Hl>b>TeGfq2k;fH&Mbf?iFVvVi(e-)B_nHWNt85vlTGl~os`7<#)Iy*BuGB5%m10Ta1dk&uCflN$} z0w3(zL=GR@EyTna!swp-z+Q%jft|sCMerEo%mX`|MSVkrq72x0fu@20*kq6i0NGp? A_y7O^ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/.README.txt.crc deleted file mode 100644 index 4f4bf2265a4689a7ad464015a6c68872d13359f1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}DG&J|_wQ5zYeI diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/.metadata.json.gz.crc deleted file mode 100644 index e1d1584337d41eb5b5454b54691ee035d5b43892..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}Biw_S_Bt6M6%E diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/README.txt b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/README.txt deleted file mode 100644 index c8b92bcc1d..0000000000 --- a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.109-b71b065e4bb6 - Created at 2023/09/08 10:40:43 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 5b3d27b2cbfff9db3ceb6448774f417fd5329a8e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}BJKyz?Fa5+Vbo diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/metadata.json.gz deleted file mode 100644 index 13ff2b0cc2090baa841fbd177dd73e6633c717b1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 366 zcmV-!0g?V6iwFP!000000F_c(OT#b}{x5m8f=c!6&6?wLnrxs;Xt6 z3uS;ODM-KprG_sHl0na((`}(H`C*cCfh{>T2W!inBN)vx4JJGpf>IS*U?xee;4SKw z&M#qqTtcV*uz0U>uWT~}fi4R$QFvgYK`7~;b(%$2K M00ILrdW~*mo#PA!Qk~<}r-o z@fl@;qmVF)C7H+U9(=bG#05oB2mFC1j*}-DBFb99ERmPMZ%@wW@XAoq+vy t*QoMkqiJB|!X_(K>h4147u*}(-OA^(kRPkfdQ};@RBwgUpW9L!@DHALUmXAd diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/.index.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/.index.crc deleted file mode 100644 index 399bfc17c871350c80f57e12e1ae79569e924312..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}Cs&B!w9O6ZZo? diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/.metadata.json.gz.crc deleted file mode 100644 index d82735252bf5ccf12488fef777e0997b07455a6a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}C6Ka$F7o5eEXp diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/index b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/index deleted file mode 100644 index 7f91fffbf24726a4068986ff0fc32f802e19088b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 104 zcmZ=}U|P=$GYN<=Y_K#nHUXWz=X-wd^tEO@gh{vGdjkr_33u>C2K;VoQ;d2XhKcN5y*87Yw4Ie~ PvXt@x0+ti5w*UYD`u|7U diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/metadata.json.gz deleted file mode 100644 index a76135d54e22562d51be66628e81e60ff62bde58..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 472 zcmV;}0Vn<+iwFP!000000F6`MYuhjo{$KpGf>zQbi#NT6jBaCNjCS-T1R={;$DJ&B z(mAaO`rjujl9RM6^u^rW_kH(E=Y4ixOJF-Nw$>`6i+qw#Nw(#+JO;DL<@EAu^67n% zUlo(Z2bdSdA|siBdoZ8`SY|-7mgClwk5NxSUikFbx8RZAgR`%sF1Vz(XgY!WL1dHT zNUV7h0o665Yj+_RT^hstwGY;++7PP^Ldhynz=RjW-!9Zos)j#kbBdhnog-aaavTJJ z>nIXjN()F2B+?rF00LRn)n$yijU#`cr|TUzHCM>$azl)MOl6PZlmt}Cq!yi}KZftS zFE^|0?BfNgkXht7HoUy1*HUwwP04aazo}sSjK){pB+IuKEJys37(8he5mU|bmzDC1 z6M*?*P@ zF6Pep!xFk_wE~4Uqx$r67(-wb(-Zo%{6EW;R#nsZiJTdb8p*?)%~%8dH`f%8Y4^x-!P OQuhyLpFQf|0{{RUb?sjO diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index a99933f31aff87f34d1bd45627f290cfd3e6620a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}CsFYun4p{oeTiD^UhL diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/metadata.json.gz deleted file mode 100644 index 8e91460c4fab3ce9c6d873d8ddee47de345bd377..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 725 zcmV;`0xJC)k=?8jwG1>0UhHYFuvGTk^HO7o<<1a?R=^13 zvyu^R%>QpcC!;5~e^LltXk3ywSGW+k(6=t{fx4xg&Q9v;RTqz{vlXW%%dV$JQJOTV zwp`4-GVZ2nmxf);dYVKL$l4jb84o*7e^@NpR`|qXT7nFb%nS0afTkvR&t4JWI%mE9 zVcN}K?`o9gelQWc12V5?rsfa3VCF`uHF3RSl&jGvzV$lxX#1BtS`=v^07 z+0_w~%2;x>wSUjE@d{Me z&9FFjqVnAj_O9*vRqUccauVO7%Ab=z+fM>-7CiG;5V077nyQDk*+$-Hku6pN9&y@S z`Gc;>+o(9@kba$fo%(TKHBml7_)A(1r}M_!ENiwHLlvD`rpN+s|4WoZnQtufqt$NF zcCQ8G2#>!qMrh(Genr|uO z&Q?Lk@w!esxcGf%cE)$xG2nUE^>Q5JVVqDjTUHsVLmnHO5+;%wGDK^MEKubCv-TA> HP6_}3TzqJT diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/parts/.part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/parts/.part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.crc deleted file mode 100644 index 85cf126ba66f60b5bd2bfd2c749c2291b0c29415..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}A9CbEq2t62k+8 diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/parts/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381 b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/parts/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381 deleted file mode 100644 index 461ecdebe686e04e47638348377e2896f84d6050..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 366 zcmWNJ%`0?K0LGtthrx|I=Gsg)7W@JC+;i?Z_xPHUk0z2xv2hkkyi1dkjU6CZ%jU+FoDnjK_{is@8uu^xsG%?@eAlB2KZZ!9P8{@!L_C~&eK?IPaP9^Q#j>@` zsT0f2XIhe4QK`Rop9*8|&sWiqLY0GF;rW4s4k^6QC~m(GwI_Aq$IHK$9v4wpyNGV- zQ?_rEB5d`gV8t5t(rvFbT2^KfwqE>eY2*(6TLlaePx@Tp%VZln4;E@iWP5nyD7Ve< E3)={DKL7v# diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/.README.txt.crc deleted file mode 100644 index 17e76b8f5b74d8de753995b0ff9788fbcbea6317..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}A_74iX0d599(9 diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/.metadata.json.gz.crc deleted file mode 100644 index d51b8848d746ac5febcdf9b99b63652a9a7c3f2e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}Erj_ml?!6Ho(Y diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/README.txt b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/README.txt deleted file mode 100644 index 440bda49da..0000000000 --- a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.109-b71b065e4bb6 - Created at 2023/08/25 10:47:38 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index d2dc39b830e972bd0f8cbab05cf11594ac345777..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}CUeE&Kog5rYF5 diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/metadata.json.gz deleted file mode 100644 index be8cfbcd3f386024b3c5ddbf688fc61596f64fa7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 295 zcmV+?0oeW@iwFP!000000F{wXPlG@Z#lOp5i-}gF4L7M64^2#p@gQl)c6Y$Fu#o*@ zf`sqx47IIbFLGMm@4YuH(=p8}U?F=p5)8=o>qF9jn6rRoJ7`&$Ztx^(34(MMc89=A z+kjji0lI)BB(pf;77}N?aGh@?cVNR=d~XbIQ-n%$({2LGtEPlZ$#XF0l`7k;@Mm#P z<34}JO$H_4Qk0Y~MkDqO2QaE1p`m)~^Bm0N!mBvCnlGXg3Yd6TrTk@y%3AEOWn|it zn5Zu_T+i=r7k^``x0|Qsko{*6zZrNnP4@KQEgEjzqbkAT78$)$V0 diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/parts/part-0 deleted file mode 100644 index d08c20b4765418d26668dcc7e1325bab2c8a85e9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 477 zcmXAkO-|cT6oth5))$cvjT>c zQ3tG}i|7_Q>k=x_1@u}Bo<2R@bI-ZwHv_)6fWITgt}m-S$Eot$#ZH#iX_eQNb1#$= z#lu2}PCSZ|^Dn2#!Qv*(lB2TD7iBsng>;4|QKE#>B9v~DmgP;I)_FEK#@mNMmCokV z{Ag4q6S6GRSw5KLS<5(#gb1`WVJQ=DhTgQk3gv0xB@b8A>tym!2q8JT0f(iDA<-r7 za;$nBJxRmO+2t5pQqhRsD#_N$M?}8%^cWT?JG5a-!D0Lam;31Y8G3q+WBC`Vt88{b z86B*L9k3TfLFmgfEK^A1*o68-3N{3@CD{>sirhX18_c33Oae)|?gnc;Vk>Cwa>>2@ z2k2m^rR)A+aeeo#uFh~yUr)TCSv3B^R%F;!1+|H>?YRZ4pbz7TaUH|>#BAk%Mtja$ neVB7*skniyskOnEY~6GsD!S5qe($s%v`N>;$}1ZeLMi+Qjf`G+ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/.index.crc b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/.index.crc deleted file mode 100644 index e911752221cc1e519f5019165a26c23a55dcce78..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}AX4DPso!5tRae diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/.metadata.json.gz.crc deleted file mode 100644 index d82735252bf5ccf12488fef777e0997b07455a6a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}C6Ka$F7o5eEXp diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/index b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/index deleted file mode 100644 index 6ba08ef5883c0e5facfd4027a17a9e3be7e129ca..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 104 zcmZ=}U|P=0FbRk-Y_K#nHUX=;&m;>pLL7)ca4<45FkmqU0AK7D3;+NC diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/metadata.json.gz deleted file mode 100644 index c951bb1eef5c6a28538ddc6c8c5164018acd0fb5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 161 zcmV;S0ABweiwFP!0000009B1K4#F@DMepJ{10ZPC9Ds!ZBoHfdTeq=54oY1_RORj% zHa6@3_n#LS_(4}n6Gs7G?doRZpiL-UU>Wz=X-wd^tEO@gh{vGdjkr_33u>C2K;VoQ;d2XhKcN5y*87Yw4Ie~ PvXt@x0+ti5w*UYD`u|7U diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/metadata.json.gz deleted file mode 100644 index 6a81de8537f587e4d4f1f9124f192692de52dcca..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 368 zcmV-$0gwJ4iwFP!000000F6>jPs1<_{VzR-sT*Sxxj~T-7X)#DLq9_>7XoK}zjiYmk`L;m{RQryzOkY$IIPm<$}&JVv)Bs8q#< z)leC7m08X2tcCVODk|_> z0`MuT>K`~=-_Xncs0VNm@_AJBPLQ!CL(v3+`W&BCG>zM}% zRLA^(Jd0|o1V6R9bkq*48=>fVEFM=pqOsU(d<6PU-_ufru8dX`w+?t09QQGl3aZ1N z-rgS~FNqC`Vb!PodVNo*-v8e>h{mGO)9&i+(_7mIBIyc##_&NO39|cx7C|U!T!1G}WUh1S+kyg6$DIhoUX0kgki!#NrSr zx?BP;LSwpGhv0)mY$A2r$&e*Tz&09ap917&<-;zLmAqI}r88`FmtB2(%&vY+Pglu2 zF*r#LNXy7m zR%fvZ0#Fq*-j0XYPJdV|S%f`_ke5P+VZ@%1%309DfK%~;0d1$(?bp+8f4kpDY3@4{ zv2mcQYGz`7*o82oS7~CBVd*3)2=A$WLj3CLa~f_?6h26Y?oE?e9+W_+*qCUC{zpB# zu7$H6<4x}7A{JM?*QPkJq9BMt=jEuBk)~SU;hyCt&9*N1l-QiP*RZ%W1NrW?y(_zZ z6uaQSOB&wcDgWQ`r~4_8%?FfrC!l&Ete*;fAB>(#qHZLSQdfUs{e9Zi=e!=br zj11|92F50qrUDFMX6DA`h5`)ljVuf-ER8u@wg6S}$ht5X$m;1H_TLvMH84Ly2=j{<<|GH6! z!C2-$qbP&wiz%WE&a#XQ0`?5vmZrugaf~0Bm+ZXF#lZ0I_)RWm9<~3B{}~wo5OXuA diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/.README.txt.crc deleted file mode 100644 index cbd98885e5aebb36400ea48f5f13666e51acb62e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}6xS#`+fk5S0Sv diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc deleted file mode 100644 index c923a35437a148b699e3b4d998d54e65d4b407fc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}8}Gr#1%w5#|FN diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/README.txt b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/README.txt deleted file mode 100644 index dfcd43478a..0000000000 --- a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.126-ee77707f4fab - Created at 2024/02/05 17:32:17 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 79ea0b74d6e0dad1fda877b28442060d1feb892f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}CU*`1Kb66YB&- diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/metadata.json.gz deleted file mode 100644 index c0e3ea8a8374d355a4419a5c6e78527167606d00..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 311 zcmV-70m%LziwFP!000000F_crPs1P>{x4it6P?*&xLH?ECMK)#VACuN-c}r?4TUAO z>3{cST{nv-Il=SA!$(gvtAJ4QY6KV%+xJb{0AG@b><-Won&mj-wE*sMLi@?^)-@oW zE(EH;L&z+Wn6;2vPak#$R)x;N4?IM}jD41PT6}-v z7dN-W|AVhB(%f&YN4gP8S$HXg<^J*Poju=^5{-Go6RI{@0$QFKQ;| zH$`91XUyC@GfZyd{J^Xi;$<(`4qeor^(B$R#I|(y$~pSARx$~br>|oD*`eXZ&fp=) e_;X!~hPu42<$<&6$GOsYfvy7qunQR&85jU2LrP!( diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/.index.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/.index.crc deleted file mode 100644 index fe10475d7274c5b1f1746b9cd69a457420912768..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}7k%G5iSt5+?&! diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/.metadata.json.gz.crc deleted file mode 100644 index 3bb6392c6f0dfc61fe6c08f05953eea9a7c13467..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}BiP^TjIw6ww4> diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/index b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/index deleted file mode 100644 index 19d8ffd823bca79ce92e27b1ed4c1f8ad454711c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 118 zcmeZbU|<-M#Y)JwNG2dEzACo2IIF4%&mxh^Athl}Pp z==#FUD0=74+r_E{9H2;2R73xVzn&avnT+g!rK9&5HCo`O&CMz4sqZJUxvhPKYS4C8 QvdUV@2M!sT6t@5X0Bs0MP5=M^ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/metadata.json.gz deleted file mode 100644 index 25de203591291eb3b84c8e43c4e4a1eef49bf670..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 409 zcmV;K0cQRmiwFP!000000F6>jPvbBU{VzTn1O<_mZ%dI7LP9_hm#VU+_9R$1w&ghO zMpgd%PSUughyyqK&3kV?cdRrP;sHFg6!gac78d#IYTUp;-V<8 zOIS0`B9<7@DQsj!D+8ntA|FwrBv_Mu>H<>aZTRde>(^k#BE?pvT^lqcY{UZ7k|5)% z6)&MSwv$zBbR+)82;KKk19h;Zx-%-v_z7{<^H1!=as$zTOVa2SkA9o89$_)5X$*<_ z{izOg^v7JuAv?ct+@5xA~+r*Zhk;Kjn_iRV9zy{5vmIHfw%9H z@@w(n($y>j6*`o|Y`zvd#avrU{O8@NqDc_b1kInO$Nig_Q(63Fu?KpZT2DJuz7F0| z*9{XpwqMuybWjg#e0{&0XUZa=8ZY6n{><+Q_1oX;Mly*4RcaF5om9S^YM2b3b7($B zzMEzK@_IG+L?4j&c;avXC(S8Kn*%s1!{n8n;@JTPa$cAx}E z?F!D8g!&Oj-`Ftcm^(inHps7w1+qc|(aML}6k?ofwe|I}jN}k5B&dTGh)|=7o!Mul z;IPQ?r4_(hga;R;Rsv&k$PDtIN?1RVexx`^ADt>L@sI067@EzZC`6FfM&T~Q>QJ;p z6~d;pOe~Ipg4GInX&S?N6CzO}>{jaGAV*dxk+SN@;$$GVYZv#CZ1}|*l+Lm1yZq_v zWB&AgetJrl%f#rqJL%e1ln^0Tg`RWNiwtT>nQ_d%J0xtX*Ve96x(?wwbnBL_O17z3 zhhT+XgF)+0>mSCK+aJNu@ z|85}`5niBvW*+wcjEoFs1{NF)*XQjKV*k2vw-7Ur%zwrgQ)C4!O^r>K?7Ypz!0_+* zO)h30HIQ0{wU6#{F&Y7FGGh3?Hq)$ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/.README.txt.crc deleted file mode 100644 index 67686cdbbe9552ee2d86123413a99d39f836e1b5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}89XLu)Dk6eR{x4it6P?*&xLH?ECMK)#VACuN-c}r?4TUAO z>3{cST{nv-Il=SA!$(gvtAJ4QY6KV%+xJb{0AG@b><-Won&mj-wE*sMLi@?^)-@oW zE(EH;L&z+Wn6;2vPak#$R)x;N4?IM}jD41PT6}-v z7dN-W|AVhB(ZC diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/parts/part-0 deleted file mode 100644 index 37557ad669655480e2c959a662e0f79bb022d73e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 168 zcmbQtz`(E=h&8tA|5aFQ%EI8(AuDfibQj}+mN)nJ_bfeepvV4~%|=D%+_u?W#>z@g zHOIs#RT#-t)Y>f_eW&KPp$4&# diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/.metadata.json.gz.crc deleted file mode 100644 index 2ed6ca7df6a25bbdf5a3977c90d8e4a375ab046e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}BhGXLtkv5}X4~ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/index b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/index deleted file mode 100644 index 8835e5ade58f16543a0b024246b988164103146c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 79 zcmY#pU|wtxEd#+H|CoNiboWWKC*>ka#n>0Rtq^=j9|H+kWbp1e{1US^Tk OzA`7Toglg$=rjO}6h|=t diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/metadata.json.gz deleted file mode 100644 index 9eaabdd550e27966b7d10dc5f226a5cbaa109b45..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 406 zcmV;H0crjpiwFP!000000F6>jO9L?w{VzF3=~mE^n^q7JM6Gx!C5*{*4Q7*&&9*LO z|GTr>O>4!2x8%)xZ$5WyZ7j+S#@cwt#O2k+bjH~S7<&ZS4G+9V4{ALp4;vlIqr|EaNA{RmVTE70V4o11?FUS3LS{&U%E!sHQO_>i4fY z(9s`rC5JS}T_{*fHjsO5&Gyr;0j3Od~XZ93OYDVoqi8=fxiAXlfnpO!+E!M_t!U z?AU%?!_z_4ukq#WcAhDVfNHpe!}>G7Bh+vIuN%lD3RI~{baztuW~^Z}c*>#q5czGE z`O9L}`$Qj*_;})Q07uO!OPe|wr;=?Sutm)TtrWYC+(!Ly!GpB>2D9<&ygCB_0H*%M A!vFvP diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 8958ab8dcedfb5cf7dae7bf83246a497f060a09b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}9L4-Fg47>Jk$GC@BTB diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/metadata.json.gz deleted file mode 100644 index 2ea2ce5a66d78cadbd44cd7143d87890918cb5d3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 672 zcmV;R0$=?fiwFP!000000NqwibJ{Qv{V%%KPQe%__*Mw*Fw@DTA?=|I;}O=Hs4+4l z3kqfY-?Osux0xh8h69+Vx6<3EUCAdR=mv;OW`e;1*u$^e`4)VO7{q&qBXJjGPbH^9 zd7P0YqJG4{*Bs)W^5Dng7TBa1Ko)2SM?Ou*G={m7J71m3nOwxVpz>%0JX9cKclt$2 zdR*lA(g^5=her>Dk{n_T$O+kRbzVObek2(cKDbp}z+ca&VW?M&q7bMswi5OkR)?ZJ z$dGPJW@0c1Bweq8mDHGSwjrP<#AZSbtQ=W`08FbRjWa>)HXa-z(TIx`l`6+3clpz| z$NcHX{QQ(Gmx(rZe>S#lDI#2~3q$9i78%s?GSirScUahTukBr@bsffa_||P(RczC- z4#Nt+3cDsKOPf{mYPAajkd0dXJ{|Y1n{hd05dVpXtU?)wk^hWZYNVA0=lmZMoZ)cb zw8MVKilw3NjKjo%Zkmya{$LVZH)bP;S&pTBDj~e5>Iw0ytIsK2qiB4Pw&a`owAwZS zm$9y>19k)Y>DUII1l10#aB;+at;Z813W69^F&z~mQdDu=+*)jsbk=yOr=q4-y;{Yk zhA38Vt=(AlgH#JPzo6kAZu0-_e!89n(Y@bkd3kuofY;d6cGYCa+bSgEO3;T;_ss7s znj(!_p|s^U#W$s2E~_rd7X*LAN<(vAxr)+UOBd?wgy>1Oc$$C3-H7#zhW=yrOv@kG zJ*#h-jx`#Zwqt?0vV&_Add@Z2H-nL5JL#Lqp9d<|FphSD>G%^#F9)4K<(0)HF>2=r zuW#8ytLNSRyw@w^Rdk6s8shpnj|)HCF|4(8W|R&EHq}#TB++81QW90*9=`yWtm0`? G2><}rtVvV= diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/parts/.part-0-2c340212-6295-462e-b4f7-e067e4839646.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/parts/.part-0-2c340212-6295-462e-b4f7-e067e4839646.crc deleted file mode 100644 index 97eeca520f0cc4d5cf692c4b6d1d22da25589a15..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}BJ+csvXM5sL!` diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/parts/part-0-2c340212-6295-462e-b4f7-e067e4839646 b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/parts/part-0-2c340212-6295-462e-b4f7-e067e4839646 deleted file mode 100644 index f6c59f243440b99ffbc0657d075a1ea0040c666e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 97 zcmeZaU|_HZVvVi(e--S1GBPkS2o{&7rDayc8=4rHS{lc@ga$A$GBZdp0Hqihr!3qp g)Zf2bh($#DKO;~KBnlH_<}qgA1qlHG*gTL906sw#F8}}l diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/.README.txt.crc deleted file mode 100644 index 0c85aed7dd2bdffe62a7118bc0b1bb29335fc176..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}D%)p>qoW6AA-$ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/.metadata.json.gz.crc deleted file mode 100644 index 093e2169db2b908ff431985014de676c02a9de98..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}AVYb-j??`@5$AC{zYW diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/README.txt b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/README.txt deleted file mode 100644 index b868a31003..0000000000 --- a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.133-4c60fddb171a - Created at 2025/03/07 11:27:39 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 7762d1a08f47596ba884f3d3b6df2a17eab4dfcd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}A{4v1l6r6Ji6y diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/metadata.json.gz deleted file mode 100644 index ab48b5f481e3f9b37c166bb18a91b543d987a076..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 376 zcmV-;0f+t{iwFP!000000F_h8PQx%1{1;yd5)`#Uxe+KQB%}s$psFI*eofsbPViDG zME-Z4n{ILFft;LKo;@SGC6bmvV)CvyDB!ctm$U{pA`!{9(2#h(z!j@FFprbCJsLim z8u;^xKoxihnVV@sHKayaX4;yXwT^p`i*}&2sN?_#Y0CiAyUgHN1fP_m%^XP~scII{ zfyp=+DvG}Zl`_Dy93)@@so~p#WDsmn(oLl<`QaqzB3pB6_SO@39>Hi)XfW~75Y?*M z05e%`1wWx~;e1QGLrcAWhRp<>1Z{MOKv2kh&_qC9YvF4Cp8q0y3T7T8?nM2(9d<~& zw5t`U;Oe|hll#$Qa;74})}r7SlBB5E7RU5NOAn^v2L+Sy_+c`%eQCq@)_?l&%s|V~ z>HLQwYO2huC_v+LApYm}j`(M>nNgWTQd!A8)e>!x#Ia}s$-9M+tgsxf%AUgd;Dk-R W#v{&8qsA!hJ?g&+5zYe9 diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/parts/part-0 deleted file mode 100644 index e4ad113e641613e6757f81438d6c8579db2ca480..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 444 zcmV;t0Ym<$0RR9H0{{RhwJ-f(5CY8%0QT5bEYNPM9sn*hu`R=7G+zX4O?{LZUN%i~ zz>+1#57NGY2flb13II!11#ot{21Wo!080Rx<=T3IonBinDkXIQh;L&$g&;FxCE?nnf!DmCQAgZ4xU`AR5Q!@(dyFmOzw zeH;ahi^FxB%U#w9HVT__90+G6F1&lEpYF79dPAfxF_8ugoE7nvRcVdJgJBEoN``Ig zv8=;#rNIHtiug`rd7H5HMVY)+ss6FR!LSG{SPevVX(XK%PZ4k3ZFYray=4`aRn4|_ z-Vd&%`_cj#03Z~BkD&+6U=;&^3_XT2KWohN*xc2Y$-*K=;Y_GpN3&4N7`RYrz^5oJ mWT*_&Y5+={GlQXW?oAt*l{pOn00000001bpFa00@0RRBXZ@UEm diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/.index.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/.index.crc deleted file mode 100644 index 4338e8fd27984ab6cf5ec8fd1ccc2599d2de2bdf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}7-ZP|^bc5zGS} diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/.metadata.json.gz.crc deleted file mode 100644 index 0a1f959fde2bb4d7be224cc2b22aba81ca800473..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9JuzIF}(6DI?P diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/index b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/index deleted file mode 100644 index a761645d2bb5aff35af44fdaf46ec5e628435e36..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 185 zcmXS7U|^^KVvVi(e-$cpnHf}A85md?w3G8Q@{3C1jVuif^^!A+4E547^AdC7lXFrN z^GXZiJ^g~?;|oCDab8givcw+Gx`7k diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/metadata.json.gz deleted file mode 100644 index a046bd79e4b782623ca77a5243fa8741523057f9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 163 zcmV;U09^kciwFP!0000009B5$4#F@DME}Keh89&7<-M#Y)13&0WdEzAC+qP=94!VpIBFnhPpeciIs$J_K5|2^AZyeldxhzit4`
Ww~JK^I6#r4sD}Oze?2+SG8x$cOHc0;YP7&lo10V8<1kEQb9HSmp>D Rm8`Rt@&R21IeoYQ007aOM{fWC diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/metadata.json.gz deleted file mode 100644 index 1c782685520781f62a3a675c36e17482f0491868..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 553 zcmV+^0@nQ>iwFP!000000F6{#Z`&{o{4ai50j+4_ASwKorR!R3>xQ=2OArW(#5n3? z$&+NaLE!&hTJcAmY{1^A@nM$6#Ex6y;MXo(m zq`C750bOL>(Un0fnOnl?0xgA=HFuYEiw&ND3gW_aN~ ze0f}K+!eT zxH_CpNUBZ71a;+^bC|rpX*)jm@PXcae(vKR-_9{bnhIzNY3swkC6re0q6DnM)C2$k-c=B- diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index c455749efaff2ebc1ab07da44a19bb1110186615..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}DHQF!RLK|K|PxDVPQ# diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/metadata.json.gz deleted file mode 100644 index f8b3e06ca32dbe1a7877f6124ad6c3d678b173b9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 873 zcmV-v1D5r!2g##Z6b7hx8x04x*~OKD%BoVRhDraa2xE* zHVB6J-*)!P)swyL1InzF6I7t7W-#ogF-=8>P%mAjUYswHyhk~quWqmlfd9QNo6oe z-tNTm_)4{x>?P;z%MS&S8ltmvcNG+f`e;r$1vI##e^-=Y43ayVMhTB9N^2Z>KLW?^Pw13v6~uj9_SSYjWkb*XB#cyG`d z-=yI|32SK$lh+Bl8$D*fe*Q>-G99!&bWpi>K*IPu+d2nHxpS%;C$3DPW3ml%Z_fP7 z_lN66`|7K0h@2z`d|vkmbb2Zol#i#7f}C<(#yGR}<<}yn`NfkM=5kH=JDgV~epU44 zb4=;VfmOxTDXcP7m7pq|Tujp;;_?~B9AzoWFf>~Q0hNWz`hGNQwyuWhlIxHK3;m4B z5Ym1He>tU@5l-1ZQ0R5L?OxgJkJ7$1J<|@7qj*`g+?xlIU}iEEau_EvJE$ZLe__1< z`NhRIt8SzKeqd+Py9QOZl>;V~>9kGUkx&5`g$h0@t~6Nj-WYOwR|J6q!js)F=K`7} zzOpi=FiEOfKMf}nOpa?4#HoAY=)L5-;PsQh0+$@e;RI^@vEo~QlJe?uTe?udLW&AS zEkhGa-UUdkDk)qfe)Yy@GIPu#$4NQ*V(i7pPg7MD@`->yBWQ3qZwzRgYkCTK?Y(0* ziD1OsLN%26&o&?RMm-??px?jj)Z1j%sCQ_iU+;HjSM`2-7Bu{BH<*#G{Skk8zNlGgWFeau0WcRLV*U$O5gJdX$cRz1*Dx``Xf16K zEr7Qcx*6Tas=Zd%s4`B%l@B0Vw~!Y-%0^}vBiWlsvDOko1OWa6C#w&DZwvqc(h;IV diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/parts/.part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/parts/.part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.crc deleted file mode 100644 index b2b31a74fe776610e36574761a6daa8b92c21f00..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9+V(>MnJ5ugJF diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/parts/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5 b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/parts/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5 deleted file mode 100644 index d154c9a31daf47c77573bb92d5865a85222a083a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 445 zcmZ3%$iVOgh&8tA|4sP9aFvJQizp+9fOc|zMt)IAypg4$plx+qHA<(`9JqWHj74;{Xez$)5{* zSQr`jUHyXH4Ghf<&5R6r4CevWGP2LRzWgCG0|P@tu34GASbAQ5uA_@?a9Dhqu_4fA z%eG_8%l@%#n#S}!fRVvLxF933IMpcLIUvF~-q@6Zk%5gBGemUcNtX7)!req1qN z>3zC$IhJuV8JcYU-fe7QVrgM! z@}%d35rh6+6+6}^OU@iKVdMyiH#1^cKYs@c!srU0Cnh_>;M1& diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/.README.txt.crc deleted file mode 100644 index a4400373154743338e43096341ff91f638053b5d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}CtlVo@ys6sZH= diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/.metadata.json.gz.crc deleted file mode 100644 index 2817ee0840e52c9da3e3b53b13e797aa07a16f40..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}Dfa{?7ye61@Xr diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/README.txt b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/README.txt deleted file mode 100644 index cd4126895e..0000000000 --- a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.109-b71b065e4bb6 - Created at 2023/08/23 14:16:34 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 18ad5aa5ab9630781295c79a0032a005a78bc93d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}Bgb8qN#=5v&4< diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/metadata.json.gz deleted file mode 100644 index 19dd3d6727d00995e9771f45114424f2530c21eb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 295 zcmV+?0oeW@iwFP!000000F{wZPlGTNh5t)mhlx(33vV(pJ}faA#s`;$bnOM}N+E4A zLBfCcUM8DNUgW8q@0@!mw`GPGz(V$HC0LN_*N3D6F=qiw_t3C#v&EArB?!`4*baf0 zrUJP<0(1dMNM>=w9VFh?!nYYOly1_zR?>yD_}*IHY|x<$x6L+iyr?v!)3B~2_b@>^ zs{8z@n<_f^ktmaoO2%L)NWc$TKYCYZ8Q8N6uj1%xzKBjJVB%eA`O6aNQtWZc$h0Fd zQD11dp5NUr{>D~sH&4qx`_CYLGw|X(+0lcysJQiyN`u2KGJ3hVWVFqXH8(j#Wo;y( tmN}0Q$6^SKJRFn}x|X=b9^o&BqZX5gl#Zrx@2#Uu_YKj;f~!md002<7jDi3F diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/parts/.part-0.crc deleted file mode 100644 index 14b34f102c45175de6be75fe39cabd9ef2743363..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}A`4$}<4~5Do%8 diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/parts/part-0 deleted file mode 100644 index a6fab3a930518526a8d53a2dd30f78591ec5a790..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 460 zcmWNM&2H0B5QWEN(MrTpoa~Syp}-bHT4*KLJT)W z9S}_dW2u!-AQwy%=!*o75yi^{emakV&~&?SA%hU4V)$*4Av$SeI2Qqpe+8us=t;*= z5<4o-f>VukEI8B)!@GtB&kGv_O231*&g_7+;R1b`07|X7iYKX_ft*o{^$7arFYLUx zIyfnNT^QID-dH(=FD`gR+^pyC0j7`>dOM8B3O4Ng04`+==nw|#5cV@2{fMsGqbzHyZ_T?^u`Swa+h%ttv*|ovlgs9l zY4|i!Uf&=iz5vGjXvH5Uk*%GVAOai0`|5qFv{(~}^;S06FGKTL1t6 diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/metadata.json.gz deleted file mode 100644 index e4b22f3bb490f95c55551b2f0b56ed6a65b01250..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 337 zcmV-X0j~ZZiwFP!000000F9ARPr@)1h5t*RD;bCo`=&D?z8Dqa3t5)Ca)HLKZMqJ! zEdB3Z#wMsQzIEq(=j`^jWm}~&KcKIaHB8Lr$$Y}u9+bX_EE!EkgZqcKWSmhpd?jY<5zNmx*D5yR(4#`=aKH8nO1!%DhzLufgLaB6#Pm~UHnd<;YLkGO7K7r|r zcsAj7!lgCR`VG)Eh7#xTkej;9L5H5|BneuJM{6~L;RcUCgvb|fJ-k?nl$@CH%ar0g zO^mcEX-j7fja4zxvh$X@qk;0Yz4wo&bU>C4W%mHzTOasHsy+Tc*TEEW;DZYE+_|)M jKSN*eI#_Av{4N{#^)!oA)Q7sa?uhsWG&;e2WC8#HWJjou diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 01d66fc81577fb38feb4079fdb5d7e5075fe702f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}AV_Wbi6bZSocXBvb`1 diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/metadata.json.gz deleted file mode 100644 index 1fd69440433b39f134f72464df9df971618240a8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 639 zcmV-_0)YJ=iwFP!000000NqtxZ`v>v{V#rPU`hBW%iEw?hlJ2psyz%L%e8Mx*5E{r z!^%+p`>w-B$Olrp$AX05+;gwbu}|=xNVWmuk*O5$3gY4C-68?*5{u9m*b#3@woHXg zY8nc%LX^h>xZx0Qe`y?c1x^<2>&9X=iUxS&9>RE{H%*_j3geILe6Vnq`gjGTGo#{2=-R`G?ZcyB;pvdK$!dlI$O3hxrV$yJ@ zhkXklhbsxiN;Quy`LW_4NI{>cF)3rsbQsHOYo4##vv!hmA*K>CBci$dF$t&D8*A5A z{U~X{=Tfq0j)(kzr(dk6K(?QE5ig35KhWB#HdAvz-jt97Re|}7XdnE}(41XVg3?r9 z*IrkCJyva!uL%B-NyFW#Zh8BkaJdatc23NYgem0O3u<8M3~i$gq-cyt))+jO-ubfwb?6QZHIzKc1SYar7X85) Z^UV~4nI`4#*E-SE+Sc&yyb15bjeF4O4sM_&b&hYNBhx6&pJWe(=*lP;-P=70W0LVT zOkNpn4<~BEC*k$aFs2N?daZWdz-PkEED9f2n-<5*7(X_kfvf0(0yIGKG9FNztT%cP f%hkRIa>((V1Vsy)gQ{La%_w06Zq9Ke&vx+#6P6~H diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/.README.txt.crc deleted file mode 100644 index a3bb76f1307c1fd446eb806c189c3219d0910f6d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}7-)8>I*U5+nlY diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/.metadata.json.gz.crc deleted file mode 100644 index dfa05f538157270698fa8e925e319c00b2cfbe24..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}DJ9*IEw%5V!)Z diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/README.txt b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/README.txt deleted file mode 100644 index c7d1359830..0000000000 --- a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.126-ee77707f4fab - Created at 2024/02/05 17:34:17 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 79ea0b74d6e0dad1fda877b28442060d1feb892f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}CU*`1Kb66YB&- diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/metadata.json.gz deleted file mode 100644 index c0e3ea8a8374d355a4419a5c6e78527167606d00..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 311 zcmV-70m%LziwFP!000000F_crPs1P>{x4it6P?*&xLH?ECMK)#VACuN-c}r?4TUAO z>3{cST{nv-Il=SA!$(gvtAJ4QY6KV%+xJb{0AG@b><-Won&mj-wE*sMLi@?^)-@oW zE(EH;L&z+Wn6;2vPak#$R)x;N4?IM}jD41PT6}-v z7dN-W|AVhB(_@s(N~NP)7ZSW1c}~wHn0DXv+LUJ#u`ESYg3dh zBqXNrPLTInm?*=p$KK|^dBh^QMZoE(Lm|Vp)i1s&Xc`9gw(pAExBS}cvn$TN*d6~@ zTxgQqWu?CRkK@j>1lc~@7$0R`knQDn=I!5{rIEI~J2aJ#_bgZ``>bl&3h^DL=lX4{ qRbx&*&STyd(@@OF%An9am8Y??cj2LpT+1YRfi49Buxl9@85jV@u1+KX diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/.index.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/.index.crc deleted file mode 100644 index c196068cba2e66a1fddf969adfc94b8eeb0789b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9MN=2k8M6&VB@ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/.metadata.json.gz.crc deleted file mode 100644 index 41cee7d3f8b97bb44926145da125f70696fb803b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}8vjFrEzn5ZwZ> diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/index b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/index deleted file mode 100644 index c7ce1f380c5c8ab6571026bd9fd73dfd6571973d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 183 zcmXS5U|=W*VvVi(e-+AQnHfY_85md?w3G8Q@{3C1jVuif^^!A+4E547^AdC7lXFrN z^GXZiJ^g~?;|_w!U=~cthB&4ZQ zO8@trU8~fmy*1zczVC9EHY`^f(-ZnqS;NF4o<&cb9YN`v$flEM@)#{15aT$W#`!#l zJ>$${jvkG~GWDoa2jNKYO(i72p5jC8kSd$Yp*d)~2QAjYS28q}%OGGW))=-VDRbEI z915lD^iZj+6wls6eP1e}aGG38B?lWHfxDLfU@f*M@Cr;wRkq{ao`Xp=!nBju5B(k$-8(pVHR;|jkG=k8hhyBJf2ch6)EV@Q=KG1Yw=>OMljsqc_g%a z_1VL#jYz1{(kv})Qrf~-4rTW~*GC!nmQ;KGzpsPI zbO1HYZ`E;-&hBpyczSF!husZ5nY85wl_hBanP(0;LP VDXsT+|5ayge*yXr;d#&k000EBxiA0# diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 59afd1815900ecc8e854c51d64b74abfbd3ec6c8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}A{e5D<{Gbn!j_BJTxE diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/metadata.json.gz deleted file mode 100644 index 49ebd019d137568e1fa2f3836c68170369f91424..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 644 zcmV-~0(<=*iwFP!000000NqtxkD5Rf{V(&nU{`Qr?;Q{n&}6rd>x0oV=YE`f56qDWwguvmnegEa{HLGyiyd&A7=*sUfp|-@XEJ0$ z(aIF5h`L%^Rpk{J&_O8&$1=~uQ|Wm@F^B(foI0cwd*C|+vH3aBZpkW+l0<+RmY z)6taCQgJDw(lEi3Qo%Y60+5YuK2FD$Jsh8p48k{I2jfsYux&Fk(;pNOYU?&M%t|bGvV`b6QyauPoi90Eqd1x%H|@KeTsN{z z^iJQLpzK%bVL}3Ok;NnX62#+_h zRMZ|>R)=q-)d|pZ5#{>kNOyELUr-HG%kAo(K=`c`yAAryruWmwg6i6RO$`N)QXh?& emqlGrVN;z&Bbn}$DJ8K*G{8UAr^)M(2mk<_v^>55 diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/parts/.part-0-cbf84037-3354-427a-98a6-b953711ae5bc.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/parts/.part-0-cbf84037-3354-427a-98a6-b953711ae5bc.crc deleted file mode 100644 index 2e373bfe9be3af3373c2e0917fc127efb4e786ac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}DIB$7~1y5B0REp+o(?dImroG2P4(|tZN>NUN1~*a8VieHer)5h9h~5|04Lwm3acBuvhtdPI0Obgd)nI9ljk_3CLa|)8*p*>|`vDIC zU_|AvRm(^aVGg2L(FNLY%1g>3SR@F+XF)LtoDxYjm{1J>00000001bpFa00@0RR9z CN9aJBf7wCr$;^ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/README.txt b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/README.txt deleted file mode 100644 index 0c0b7e087b..0000000000 --- a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.133-4c60fddb171a - Created at 2024/12/04 10:48:02 \ No newline at end of file diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/_SUCCESS b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 889a8c9c90ea1ce66ce1ada438b4ea230edd0a0c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}A7v5qjTwRzMj5BYgz` diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/metadata.json.gz deleted file mode 100644 index 04ec6d3b2776a3f511a54ab0df68a86a1124ff7b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 613 zcmV-r0-F6FiwFP!000000M%AakJ>O0{V%yTmD;U#RdFj_R$8f5r3Lj+gd$@P!Mll_ z+7ncY^4~iqA+b%6*jtxV{NBuCKKufQM6nW*N%F%8Fd&|O-p?!GH%XdgFR)K0%N0Fw zEr2^bnH)|Di+u&+Yfn%VIEa{cvnjJMM^ke=R1CLv5q%MdeDDOhn)o3i&6-N(|H-Te2zuWLYteJ=CN-FqSHNtL1~L z%h%OD1U7yL;Wl*+N^*&NqP`fz_RG^(u2N<#7$nJau8Bu5sLmfB7Fk8NZ$$BXjm!xy zm_;xbqMCS;ek0*L1E!y;9eISgAcL*OMfjAcX&ek%z|zI=aud)+qUJN@-zGZ00Hp;6h=S=Yr0@fm&>{&6_iC?ykkI;vr!hrzzo49UUpz(XqO7e6O_H#IPEzM}7VjSfBEZJ{9J! zsxXAP%P`_hU6*xD8OyY_r-JJoZ|C<)exEnvx3uzk{#8hL?G06(ez~_Fwb{Yy!sXj! zpK9jGO-;XVcYUWjqH$GWz+Q-n8{Svd0t;pwM8Mq}V|N8Z2{g zfCkI52FtF@q5>Q;%>##w=ETOK{D3Yv9x^S{JcQ&2B;mm6Z%=OI2Xw#_GJ#EKCO8um z+5~4Jz_ppquA)cJw6ndT~)(NcPLea4F+Ai)+t3`zo9JE7U7|40rhXsetCvSGzMU- z3bnPzqGDMp76vTTWu&+QU&50Q2XANDxoZeo`u@PZz8<%8o!CK*uFYw@%lC~Mf(YJn zmQK(9PRVy>EXml`1G)1LH#l!M@haYO9oOctDV?&Z*AP?cHf0VuCfv>ztn=-4&Nl`I zsw&QL@;*V!h2q7|X-Lp-ctKF5y>x;W^RL$ep6@j9F?moOl#f2ywxPY$ zzwlbHRBiYnnSbfaAF(f@;y~s3C&haY!&z^Sc7r-bcdf*t6+Pd;c_GQ6Uy3Ey7&o;y z?C6>!aW)M+HY6wBJ7=kU$9Q|We$NPP#`5iBlqPLXa-FAh4a&y5@e9NMoId}4YYJ-`#&qvN0K@c zYa&_eq%yN%E^KYtKr)!j1Ol>c6UOQ=lV!22X|n*rWiGU3q8$hU^&(?)jBUtKHz+O@ zoed}yLdsNVA34(G=7W*Z5!sMBnS~b8*=#hF4+hdgL^2YHBeM}U;z(?Sb#y|VP)H*; zm_`x`X+$IoPltnvNNkj87)WBH51kDXXCpgWCln$YZ7>kV1{29N$-z1YEGG2Dn-h3% zYj-zsP^zNWS3&moO?ouSgrvcvUHpjUDB$dyaVUDw_h7&gVILS25-lNII5JH{W;kSD zF;B2V9MM=(z|z}*`>3Hn#SlU>A|n&DAW6~~KtjWm73&fJb77d}oOFmrhGt<|m0HaK z7^-=WsYdKs?K-Ans^>^oDO)wt>7Pp_q$SOYq`^V*T!2ZM(!%4(Uf*d6%M*^4I3RC! z%=?>QPKj-b)L%5Ie38;skmvP4*3cQc;N@;@=y(#4paE3U(v*yVQXRk|R0&Wmu?fix z5T$Cpd=bb7hZs{alBl}q_u8{&n_VKsH(=hn7YlP%01D8Y#X)(00G$e)TP|BdYrD^s z@<}Q1qt}Sy2!u|A6Fh1CoB!9fB>a)A!{Q-z87Joz_cjD`-|G5iMCmb X*?g)E000000000ewJ-f300961*ux@- diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/.index.crc b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/.index.crc deleted file mode 100644 index 6f23e8d1f3a431b8a220a71c12396c562f411c17..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}E?cINcTi6UGBt diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/.metadata.json.gz.crc deleted file mode 100644 index 3ed7199fef2f1f6598bae816845573ed5651c089..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}D&%Gv^8b68Hm= diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/index b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/index deleted file mode 100644 index 9ac97b72472b3c37b7bb591d5b42b322feaac8df..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 67 zcmb1TU|6+dM z5-t?_h6d}KKFY&%XS?%8=ha*&Ytdqq>kw4BbC8LT5dSA9C!)FlT2+!{L&Lq(BMm1*}7@lI-?wd2^p86-Rjjlk2OStHpBp z;iI^{F7kZ8xLfcfO)L~(K?!iof@FqM<;f@1Q;6@qJhcXVWfK;MQtCaI*_v!4=n$G* zTvS+AOlalEa#ZA1OK*VHNIFnZm8p=ZUbv0Re6L(#CX7<)n&%U!0*Qa8NKI!dp;B{m zw#3?SK?BgGLlpnNQBY0&R+oX&o%@XEp-WIGklO5%wY=RP#dj%p4lH?zDc;RD7j)L& z9yZKa$Y&Zq8{$D?!ySRW!06_=9BodM8L%;%@gq*enIrE7qCoCH31A;$I zKe3UwM;&H%=KYMxy!m=ombzg6Hu=hYN5L53XpF4l{mxNbNV7{DlXD7YfVM-6;8Hq> z=bRd=pFj{R8_eU``|mtMn{A~QjY~ZrB)DB_^=tfYU&(%i-*;ahSJm?VedNuyf{vXA zA^WVg1g-)q@fl=$q1bi)whLVwtw51WvO{vb=>1>C>+k?P{{FIxl(hlTzaitp5&4}* z_3PXFda`da+z)36)E{Xxco+nJ#nP&;%*7&qyVwOCt4)}5m?(_k&@*M_=m?b{*?xx> b@+9+#L+8&j_V;gJ-`f2HPQg`-bOitafAk8= diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 2adae799e75b8f559a356c5c1561e169c5753c06..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}6xoo$vHNezGzEAO!?| diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/metadata.json.gz deleted file mode 100644 index 9e1234feb99cee8446dc09bc2d6aafe8952e1a47..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 585 zcmV-P0=E4hiwFP!000000Ns{RZ`v>vh5w76HZ61o63X(nFg77|(*)HXRv~2U8}M*# zWvfj~kwjcR=<4-q-O_POWsNoGeBeE<@Zk>c!IEY__I$sTZIGi-(EU5~%XB6f2P1rd8*g z`?+tf($=kizWOvCEW7>JrEB4dg+eXrj(;^g^_y?f6ijVqx5K2}=_kjffUu>I{G=qo z&Q@q#DVlq6B7R{(FN(Uo^KIWV%k!-l?!%NoAL=`u|6mB=DXC=`9kd)81M({l0QIY@ zw|Ti&aSCaU#y|0L%0?q6t)>EyU7XqLKqp19X+&m4a+|Ph&MkptCmI?{8+K+t>Vxc@c zbcKxprVVdy>Ov072~^lSPhQVmPklM5O;P?32~i z_+Z8Dk6#Lg(UmRDXKd%dFp1mYO~31oe-P0B4hseZe{lke|BqvYVsgU~?7S_^4W%~p X4qkCS3ANUEBjDqjHr5hVi0 diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/parts/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15 b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/parts/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15 deleted file mode 100644 index a40e1db48fb532112accb2e4911a9779977d53e4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 51 vcmb1RU|^cbm614+4 diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/README.txt b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/README.txt deleted file mode 100644 index a259c4fd70..0000000000 --- a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.133-4c60fddb171a - Created at 2024/12/04 10:46:25 \ No newline at end of file diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/_SUCCESS b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 782d7d3e899d51645ff6eb159812b9cd77122b9d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}AW={=yvq6|Dq+ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/metadata.json.gz deleted file mode 100644 index 369b04d91accc5ed605b5343b6d440e356c500c5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 239 zcmV;bb<-N-|w&!*WMQNahQkOo3!Xhot@sOphhu z+Xb+TL}JoPl@63eUzu1pLJq&=T3c1?HTk=x@2Pt24%I(ugW1{~uPiDr%+9*ajcFxf zX201m==Xbr$Jl3xc#b}q<0q5e-zOV6{E1TgIJP+jE(+xG(g7*#YogtTR8ct;R6~di pc`b*?;m=^5$t&cd&WNj_ipltq5OD(J diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/index b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/index deleted file mode 100644 index c3694ceb093c8780f2cc1e7fac6ed3e30f03acad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 71 zcmb1VU|b%7 diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/metadata.json.gz deleted file mode 100644 index 172607a22a25fc75d8e6556b90df6cfebbf5d65d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 185 zcmV;q07m~GiwFP!0000009B5`3c@fDME_+^3OQ6;OU+FLJt!zDUc^JJ+a?$iNw#1p z{dc$CyetDV^LBb@jKwR4XuJbimRV5=cm(BYZCtxiHTeLt$d;u5ji%KEaxUP7>5|?E z63!I5ntJQ&F3SCQC)?9n=hajwYtdqqs}NMWagd4k0R$-LJSpOr|B%shJ4_MeOn`Au n{2Uc&4*WE`!FxJ1O+=oNFPKD3eRPCe>4)J9c<2+M&j0`bP##cr diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/metadata.json.gz deleted file mode 100644 index b33ebb128cacd51c7ad9f594b4f455e32b1f85a5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 309 zcmV-50m}X#iwFP!000000F9BsYlAQt#s8O_HgKiZ(U#mc2Rm$s(T-iDNQ|Ev`ZbXx zT@lIuzL;rCyUQ-3@BQB6CHABe2>CZ?Ba|d`xk?u6h%`*#MI`gtViqUK!y{kLtFkQO zRm@04G*qBLaxl3D%v)v~4_{P|SXTIY+qK}G&*7u5Q56j7*6Jg-y^HqH(F_$onus9+ zg4sn{ohBeS^nG~Vy>8MbS>I0_ebSnBnFocecabnTialT#mZ%z};PhE31SSXbuj-ti z$km}$5~MYR_T+b6c$MrhtloURq!Y4jfro>iPdo9>tLpUsT!4M3W!ehsuQ})HIBpmV zJ|-&-&TZMmXYp?6(W(W{!&Sps95dHcDU218TkPJTgOX5xr$Nv{UZCcVL{0NXYV5~x_rm0kWScM{EU%dF_1VRX@R4VnQJRov+zVCf6AIa>AqEA3VGF2R&f#3i7HroJe5r;%;*pcv& z$b@ZEe5O)@gp|xN#S#w87Lss(?C`ML03VkLtO6Ut%$LcCrZ6*l%gjD+7gi%IQAR97 ztP+aqowG?&VML-ZGF7JC%AA9AI2me9caO))xg*j@iZ#exw4r8|EI|o|y^}-&RW{oY z&g^J^52Kw8b!Wdn-QJH^?f&Odw9v#tp_VTwHVanu7TY)mUEA2*WaPK{qf^QuY$YVG zl#Fn)H42{L)Y%K@-9+Xu=IaM-lgxl>$>ObIc>7?f{~AiE-vkj7f!x|({|3gWW+Y# zP;I^odo|;2Z693EWuEWx?qJ|D=m&169kksA3BKh?x|YBDA_7* z&+kcUu!A*L-23>WKp0ioO8Yp@5n&Sgp5FuU2`K)`Mgza=`$50f q>j7(S^G-rBx#dWAJ{y{cGB&XyHj-I8)EI5Gvg|(qtHt};2mk=}u_yZg diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/parts/.part-0-c010d599-ce86-4363-be96-0c7b7120788b.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/parts/.part-0-c010d599-ce86-4363-be96-0c7b7120788b.crc deleted file mode 100644 index 4fde886a9f30910df5b7f6ba0860b6ee468a3743..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}Bi|D%uYK6Pg2L diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/parts/part-0-c010d599-ce86-4363-be96-0c7b7120788b b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/parts/part-0-c010d599-ce86-4363-be96-0c7b7120788b deleted file mode 100644 index ad1aada68094e5285a4bacc08c6027849d766b75..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 52 ycmb1VU| diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/README.txt b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/README.txt deleted file mode 100644 index 92647e9f70..0000000000 --- a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.133-4c60fddb171a - Created at 2024/12/04 10:46:46 \ No newline at end of file diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/_SUCCESS b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 782d7d3e899d51645ff6eb159812b9cd77122b9d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}AW={=yvq6|Dq+ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/metadata.json.gz deleted file mode 100644 index 369b04d91accc5ed605b5343b6d440e356c500c5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 239 zcmV;bb<-N-|w&!*WMQNahQkOo3!Xhot@sOphhu z+Xb+TL}JoPl@63eUzu1pLJq&=T3c1?HTk=x@2Pt24%I(ugW1{~uPiDr%+9*ajcFxf zX201m==Xbr$Jl3xc#b}q<0q5e-zOV6{E1TgIJP+jE(+xG(g7*#YogtTR8ct;R6~di pc`b*?;m=^5$t&cd&WNj_i diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/index b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/index deleted file mode 100644 index 33d6653b421ae863af68c0e44412eb43e93157b2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112 zcmZ=~U|?_pVvVi(e-)grGBU6*Gcd3)uq0;`87}f?Vs>~Px2xN#fe!QT~ r#OTiGsNS~yB9Ilr=x)s{!ouAAWR8I%&=?VzQ6dW&7#JCWJT#L4V@w!F diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/metadata.json.gz deleted file mode 100644 index 521ca22d19aaad9700d820d725fb93cc43d0d61f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 184 zcmV;p07w5HiwFP!0000009B5`3c@fDME_+^3ON+5wdN**9uyQ6FXADs+a?$iNw#1p z{dc$CybJ>~`(}G+jKv#OntTA6XRAVhM$>8nITP^0bWLvr z2^R`oO}+JX7v*8p$?m+-^KvSbwP-QQbqFfmxyZ!F00NYAo)q!qKV;0@9#iB+0meP? mb5f+a@bjDo@9Eey5&4bff=R^Gsw3o+ei**=>PSz|0002eL{k6& diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/metadata.json.gz deleted file mode 100644 index 41f6e11f7c95fcfdd6ed18cb5e348cdee1fdb338..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 316 zcmV-C0mJ?uiwFP!000000F9ARYr-%Th5t*QHt57!r;@kL!5+4Up<^#1A*Q!&SWHS1 zC#B@S?>1`{1z|6tIp;fv-rSZHQUUt`>!i_y&gR)DzoE?uDv}7--hHH8oE(N99+Vc z0tIfHSaTeJ=+O7>yn9_Ot88&UaP-Mq-fR$*y4)txvC^}Y)nEl^PoIrZz;&-@H}uCF zdOE%~LcoE9+1Ey+>IiMg-z@P{^L;nt>g#1WAnO`Le5TXe417S<9RHt-FotSwz4W*g zCd{G#hrZxru$IxeE*tpGbldejWAOvjd^Krc~ O8h!zCg&M$D0ssJD$eEV_ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index c844290c9d860e7c72f5cd42567c707d4728d74e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}D&^#Q)F_J|0B?CT|4S diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/metadata.json.gz deleted file mode 100644 index 8a31cf2f6030a57e4e4c0e4aaf6bff0097ce778a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 613 zcmV-r0-F6FiwFP!000000Ns{RZ`v>r$G?l8HZAEA0$G|jV5~x_rm0kWScM{EU%+eX zAjeE)DBr#31VTuNR4VnQJVf~JeE0u5KPlc3NuPj(WGXp41Hb$IeYOVHBNmAsVN1dX zlB8^t;hH81B#@CgrdY~>nL-lo4jt~dYvAK5fmL8anE5gp(F|rEp_BZYfucN#~SsI3A7EY*ZW`eqPdr6juIE$ZPv1|e=&$-+Y3C`?g!tvO=&N%IcLsw?67{oS#eMJ zdq?FGM(IX!vwuf2jU9}!@>YR=N`!GLTPYXi91td<>$rVCKVCd={%}0Tk|Du&PC)Wk zHoA5PuImSlUiVmYQ*;uNB^!=(7uwJ~RI!N}Vj{V*L$y|BR-V5ACP?lSI0*m%YaA%8 diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/parts/.part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/parts/.part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.crc deleted file mode 100644 index b8e95019c5c60153d3e87216b4795cf83e0058b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}D&*VKNZ_5!C|W diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/parts/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/parts/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc deleted file mode 100644 index f43efce500198d7dbc95748437a7f1a9b5461e89..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 101 zcmWGwU|OWHIspnfdK$35fxwn diff --git a/hail_search/queries/__init__.py b/hail_search/queries/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/queries/base.py b/hail_search/queries/base.py deleted file mode 100644 index 518866165a..0000000000 --- a/hail_search/queries/base.py +++ /dev/null @@ -1,1314 +0,0 @@ -from aiohttp.web import HTTPBadRequest, HTTPNotFound, HTTPInternalServerError -from collections import defaultdict, namedtuple -import hail as hl -import logging -import os - -from hail_search.constants import AFFECTED_ID, ALT_ALT, ANNOTATION_OVERRIDE_FIELDS, ANY_AFFECTED, COMP_HET_ALT, \ - COMPOUND_HET, GENOME_VERSION_GRCh38, GROUPED_VARIANTS_FIELD, ALLOWED_TRANSCRIPTS, ALLOWED_SECONDARY_TRANSCRIPTS, HAS_ANNOTATION_OVERRIDE, \ - HAS_ALT, HAS_REF,INHERITANCE_FILTERS, PATH_FREQ_OVERRIDE_CUTOFF, RECESSIVE, REF_ALT, REF_REF, MAX_LOAD_INTERVALS, \ - UNAFFECTED_ID, X_LINKED_RECESSIVE, XPOS, OMIM_SORT, FAMILY_GUID_FIELD, GENOTYPES_FIELD, AFFECTED_ID_MAP, FILTERED_GENE_TRANSCRIPTS - -HAIL_SEARCH_DATA_DIR = os.environ.get('HAIL_SEARCH_DATA_DIR', '/seqr/seqr-hail-search-data') -IN_MEMORY_DIR = os.environ.get('IN_MEMORY_DIR', HAIL_SEARCH_DATA_DIR) - -# Number of filtered genes at which pre-filtering a table by gene-intervals does not improve performance -# Estimated based on behavior for several representative gene lists -MAX_GENE_INTERVALS = int(os.environ.get('MAX_GENE_INTERVALS', MAX_LOAD_INTERVALS)) - -# Optimal number of entry table partitions, balancing parallelization with partition overhead -# Experimentally determined based on compound het search performance: -# https://github.com/broadinstitute/seqr-private/issues/1283#issuecomment-1973392719 -MAX_PARTITIONS = 12 - -logger = logging.getLogger(__name__) - - -PredictionPath = namedtuple('PredictionPath', ['source', 'field', 'format'], defaults=[lambda x: x]) -QualityFilterFormat = namedtuple('QualityFilterFormat', ['scale', 'override'], defaults=[None, None]) - - -def _to_camel_case(snake_case_str): - converted = snake_case_str.replace('_', ' ').title().replace(' ', '') - return converted[0].lower() + converted[1:] - - -class BaseHailTableQuery(object): - - DATA_TYPE = None - KEY_FIELD = None - LOADED_GLOBALS = None - - GENOTYPE_QUERY_MAP = { - REF_REF: lambda gt: gt.is_hom_ref(), - REF_ALT: lambda gt: gt.is_het(), - COMP_HET_ALT: lambda gt: gt.is_het(), - ALT_ALT: lambda gt: gt.is_hom_var(), - HAS_ALT: lambda gt: gt.is_non_ref(), - HAS_REF: lambda gt: gt.is_hom_ref() | gt.is_het_ref(), - } - MISSING_NUM_ALT = -1 - - GENOTYPE_FIELDS = {} - COMPUTED_GENOTYPE_FIELDS = {} - GENOTYPE_OVERRIDE_FIELDS = {} - GENOTYPE_QUERY_FIELDS = {} - QUALITY_FILTER_FORMAT = {} - POPULATIONS = {} - POPULATION_FIELDS = {} - POPULATION_KEYS = ['AF', 'AC', 'AN', 'Hom', 'Hemi', 'Het'] - PREDICTION_FIELDS_CONFIG = {} - ANNOTATION_OVERRIDE_FIELDS = [] - SECONDARY_ANNOTATION_OVERRIDE_FIELDS = None - - GENOME_VERSION = GENOME_VERSION_GRCh38 - GLOBALS = ['enums'] - TRANSCRIPTS_FIELD = None - CORE_FIELDS = [XPOS] - BASE_ANNOTATION_FIELDS = { - FAMILY_GUID_FIELD: lambda r: hl.set( - r.family_entries.filter(hl.is_defined).map(lambda entries: entries.first().familyGuid)), - 'variantId': lambda r: r.variant_id, - } - ENUM_ANNOTATION_FIELDS = { - 'transcripts': { - 'response_key': 'transcripts', - 'empty_array': True, - 'format_array_values': lambda values, *args: values.group_by(lambda t: t.geneId), - }, - } - LIFTOVER_ANNOTATION_FIELDS = { - 'liftedOverGenomeVersion': lambda r: hl.or_missing(hl.is_defined(r.rg37_locus), '37'), - 'liftedOverChrom': lambda r: hl.or_missing(hl.is_defined(r.rg37_locus), r.rg37_locus.contig), - 'liftedOverPos': lambda r: hl.or_missing(hl.is_defined(r.rg37_locus), r.rg37_locus.position), - } - - SORTS = { - XPOS: lambda r: [r.xpos], - 'family_guid': lambda r: [ - hl.int(r.family_entries.find(hl.is_defined).first().familyGuid.first_match_in('(\d+)').first()) - ], - } - - @classmethod - def load_globals(cls): - ht_path = cls._get_table_path('annotations.ht') - try: - ht = hl.read_table(ht_path) - except Exception: - return None - - ht_globals = hl.eval(ht.globals.select(*cls.GLOBALS)) - cls.LOADED_GLOBALS = {k: ht_globals[k] for k in cls.GLOBALS} - return cls.LOADED_GLOBALS - - @classmethod - def _format_population_config(cls, pop_config): - base_pop_config = {field.lower(): field for field in cls.POPULATION_KEYS} - base_pop_config.update(pop_config) - base_pop_config.pop('sort', None) - base_pop_config.pop('sort_subfield', None) - return base_pop_config - - def annotation_fields(self, include_genotype_overrides=True): - annotation_fields = { - GENOTYPES_FIELD: lambda r: r.family_entries.flatmap(lambda x: x).filter( - lambda gt: hl.is_defined(gt.individualGuid) - ).group_by(lambda x: x.individualGuid).map_values(lambda x: self._get_sample_genotype( - x, r, select_fields=['individualGuid'], include_genotype_overrides=include_genotype_overrides, - )), - 'populations': lambda r: hl.struct(**{ - population: self.population_expression(r, population) for population in self.POPULATIONS.keys() - }), - 'predictions': lambda r: hl.struct(**{ - prediction: self._format_enum(r[path.source], path.field, self._enums[path.source][path.field]) - if self._enums.get(path.source, {}).get(path.field) else path.format(r[path.source][path.field]) - for prediction, path in self.PREDICTION_FIELDS_CONFIG.items() - }), - } - annotation_fields.update(self.BASE_ANNOTATION_FIELDS) - annotation_fields.update(self.LIFTOVER_ANNOTATION_FIELDS) - annotation_fields.update(self._additional_annotation_fields()) - - prediction_fields = {path.source for path in self.PREDICTION_FIELDS_CONFIG.values()} - annotation_fields.update([ - self._format_enum_response(k, enum) for k, enum in self._enums.items() - if enum and k not in prediction_fields - ]) - - return annotation_fields - - def _get_sample_genotype(self, samples, r=None, include_genotype_overrides=False, select_fields=None, **kwargs): - sample = samples[0] - return self._select_genotype_for_sample(sample, r, include_genotype_overrides, select_fields) - - def _select_genotype_for_sample(self, sample, r, include_genotype_overrides, select_fields): - return sample.select( - 'sampleId', 'sampleType', 'familyGuid', 'filters', *(select_fields or []), - numAlt=hl.if_else(hl.is_defined(sample.GT), sample.GT.n_alt_alleles(), self.MISSING_NUM_ALT), - **{k: sample[field] for k, field in self.GENOTYPE_FIELDS.items()}, - **{_to_camel_case(k): v(sample, k, r) for k, v in self.COMPUTED_GENOTYPE_FIELDS.items() - if include_genotype_overrides or k not in self.GENOTYPE_OVERRIDE_FIELDS}, - ) - - def _additional_annotation_fields(self): - return {} - - def population_expression(self, r, population): - pop_config = self._format_population_config(self.POPULATIONS[population]) - pop_field = self.POPULATION_FIELDS.get(population, population) - return hl.struct(**{ - response_key: hl.or_else(r[pop_field][field], '' if response_key == 'id' else 0) - for response_key, field in pop_config.items() if field is not None - }) - - def _get_enum_lookup(self, field, subfield, nested_subfield=None): - enum_field = self._enums.get(field, {}) - if subfield: - enum_field = enum_field.get(subfield) - if nested_subfield: - enum_field = enum_field.get(nested_subfield) - if enum_field is None: - return None - return {v: i for i, v in enumerate(enum_field)} - - def _get_enum_terms_ids(self, field, subfield, terms, nested_subfield=None): - if not terms: - return set() - enum = self._get_enum_lookup(field, subfield, nested_subfield=nested_subfield) - return {enum[t] for t in terms if enum.get(t) is not None} - - def _format_enum_response(self, k, enum): - enum_config = self.ENUM_ANNOTATION_FIELDS.get(k, {}) - value = lambda r: self._format_enum(r, k, enum, ht_globals=self._globals, **enum_config) - return enum_config.get('response_key', _to_camel_case(k)), value - - @staticmethod - def _camelcase_value(value): - return value.rename({k: _to_camel_case(k) for k in value.keys()}) - - @classmethod - def _format_enum(cls, r, field, enum, empty_array=False, format_array_values=None, **kwargs): - if hasattr(r, f'{field}_id'): - return hl.array(enum)[r[f'{field}_id']] - - value = r[field] - if hasattr(value, 'map'): - if empty_array: - value = hl.or_else(value, hl.empty_array(value.dtype.element_type)) - value = value.map(lambda x: cls._enum_field(field, x, enum, **kwargs, format_value=cls._camelcase_value)) - if format_array_values: - value = format_array_values(value, r) - return value - - return cls._enum_field(field, value, enum, **kwargs) - - @classmethod - def _enum_field(cls, field_name, value, enum, ht_globals=None, annotate_value=None, format_value=None, drop_fields=None, enum_keys=None, include_version=False, **kwargs): - annotations = {} - drop = [] + (drop_fields or []) - value_keys = value.keys() - for field in (enum_keys or enum.keys()): - field_enum = enum[field] - is_nested_struct = field in value_keys - is_array = f'{field}_ids' in value_keys - - if is_nested_struct: - annotations[field] = cls._enum_field(field, value[field], field_enum, format_value=format_value) - else: - value_field = f"{field}_id{'s' if is_array else ''}" - drop.append(value_field) - enum_array = hl.array(field_enum) - if is_array: - annotations[f'{field}s'] = value[value_field].map(lambda v: enum_array[v]) - else: - annotations[field] = enum_array[value[value_field]] - - if include_version: - annotations['version'] = ht_globals['versions'][field_name] - - value = value.annotate(**annotations) - if annotate_value: - annotations = annotate_value(value, enum) - value = value.annotate(**annotations) - value = value.drop(*drop) - - if format_value: - value = format_value(value) - - return value - - def __init__(self, sample_data, sort=XPOS, sort_metadata=None, num_results=100, inheritance_mode=None, - override_comp_het_alt=False, **kwargs): - if not self.LOADED_GLOBALS: - raise HTTPInternalServerError(reason=f'No loaded {self.DATA_TYPE} data found') - - self.unfiltered_comp_het_ht = None - self._sort = sort - self._sort_metadata = sort_metadata - self._num_results = num_results - self._override_comp_het_alt = override_comp_het_alt - self._ht = None - self._comp_het_ht = None - self._inheritance_mode = inheritance_mode - self._has_secondary_annotations = False - self._is_multi_data_type_comp_het = False - self.max_unaffected_samples = None - self._n_partitions = min(MAX_PARTITIONS, (os.cpu_count() or 2)-1) - self._load_table_kwargs = {'_n_partitions': self._n_partitions} - self.entry_samples_by_family_guid = {} - - if sample_data: - self._load_filtered_table(sample_data, **kwargs) - - @property - def _has_comp_het_search(self): - return self._inheritance_mode in {RECESSIVE, COMPOUND_HET} - - @property - def _globals(self): - return self.LOADED_GLOBALS - - @property - def _enums(self): - return self._globals['enums'] - - def _load_filtered_table(self, sample_data, intervals=None, annotations=None, annotations_secondary=None, overlapped_families=None, **kwargs): - parsed_intervals = self._parse_intervals(intervals, **kwargs) - parsed_annotations = self._parse_annotations(annotations, annotations_secondary, **kwargs) - self.import_filtered_table( - *self._parse_sample_data(sample_data, overlapped_families=overlapped_families), parsed_intervals=parsed_intervals, raw_intervals=intervals, parsed_annotations=parsed_annotations, **kwargs) - - @classmethod - def _get_table_path(cls, path): - return f'{cls._get_table_dir(path)}/{cls.GENOME_VERSION}/{cls.DATA_TYPE}/{path}' - - @classmethod - def _get_table_dir(cls, path): - return IN_MEMORY_DIR if path == 'annotations.ht' else HAIL_SEARCH_DATA_DIR - - def _read_table(self, path, drop_globals=None, skip_missing_field=None): - table_path = self._get_table_path(path) - if 'variant_ht' in self._load_table_kwargs: - ht = self._query_table_annotations(self._load_table_kwargs['variant_ht'], table_path) - if skip_missing_field and not ht.any(hl.is_defined(ht[skip_missing_field])): - return None - ht_globals = hl.read_table(table_path).index_globals() - if drop_globals: - ht_globals = ht_globals.drop(*drop_globals) - return ht.annotate_globals(**ht_globals) - return hl.read_table(table_path, **self._load_table_kwargs) - - @staticmethod - def _query_table_annotations(ht, query_table_path): - query_result = hl.query_table(query_table_path, ht.key).first().drop(*ht.key) - return ht.annotate(**query_result) - - def _parse_sample_data(self, sample_data, overlapped_families=None): - """ - Organizes sample_data by project, sample type, and family in a nested dictionary format. - Returns a tuple containing: - - project_samples (defaultdict): {: {: {: [, ...]}}} - - num_families (int): The number of unique families in the sample data. - """ - if not self._is_multi_data_type_comp_het: - overlapped_families = None - - families = set() - project_samples = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) - for s in sample_data: - if overlapped_families is not None and s['family_guid'] not in overlapped_families: - continue - families.add(s['family_guid']) - project_samples[s['project_guid']][s['sample_type']][s['family_guid']].append(s) - - num_families = len(families) - logger.info(f'Loading {self.DATA_TYPE} data for {num_families} families in {len(project_samples)} projects') - return project_samples, num_families - - def _import_and_filter_multiple_project_hts( - self, project_samples: dict, n_partitions=MAX_PARTITIONS, **kwargs - ) -> tuple[hl.Table, hl.Table]: - """ - In the variant lookup control flow, project_samples looks like this: - {: {: {: True}, : {: True}}, : ...} - In the variant search control flow, project_samples looks like this: - {: {: {: [, , ...]}, : {: []} ...}, : ...} - """ - entries_hts = self._load_project_hts(project_samples, n_partitions, **kwargs) - filtered_project_hts = [] - filtered_comp_het_project_hts = [] - for ht, project_families in entries_hts: - ht, comp_het_ht = self._filter_single_entries_table(ht, project_families, is_merged_ht=True, **kwargs) - if ht is not None: - filtered_project_hts.append(ht) - if comp_het_ht is not None: - filtered_comp_het_project_hts.append(comp_het_ht) - - return self._merge_filtered_hts(filtered_comp_het_project_hts, filtered_project_hts, n_partitions) - - def _load_project_hts(self, project_samples, n_partitions, skip_missing_field=None, **kwargs): - # Need to chunk tables or else evaluating table globals throws LineTooLong exception - # However, minimizing number of chunks minimizes number of aggregations/ evals and improves performance - # Adapted from https://discuss.hail.is/t/importing-many-sample-specific-vcfs/2002/8 - chunk_size = 64 - all_project_hts = [] - project_hts = [] - sample_data = {} - skipped = 0 - - for project_guid, project_sample_type_data in project_samples.items(): - for sample_type, family_sample_data in project_sample_type_data.items(): - project_ht = self._read_project_data(project_guid, sample_type, skip_missing_field=skip_missing_field) - if project_ht is None: - skipped += 1 - continue - project_hts.append(project_ht) - sample_data.update(family_sample_data) - - if len(project_hts) >= chunk_size: - ht = self._prefilter_merged_project_hts(project_hts, n_partitions, **kwargs) - all_project_hts.append((ht, sample_data)) - project_hts = [] - sample_data = {} - - if project_hts: - ht = self._prefilter_merged_project_hts(project_hts, n_partitions, **kwargs) - all_project_hts.append((ht, sample_data)) - - if skipped: - logger.info(f'Skipped {skipped} projects with no matched variants') - - return all_project_hts - - def _import_families_tables(self, project_samples: dict, num_families: int, **kwargs): - if num_families == 1 or len(project_samples) == 1: - project_guid, project_sample_type_data = list(project_samples.items())[0] - return self._import_and_filter_entries_ht( - project_guid, num_families, project_sample_type_data, **kwargs - ) - else: - return self._import_and_filter_multiple_project_hts(project_samples, **kwargs) - - def import_filtered_table(self, project_samples: dict, num_families: int, **kwargs): - families_ht, comp_het_families_ht = self._import_families_tables(project_samples, num_families, **kwargs) - - if comp_het_families_ht is not None: - self._comp_het_ht = self._query_table_annotations(comp_het_families_ht, self._get_table_path('annotations.ht')) - self._comp_het_ht = self._filter_annotated_table(self._comp_het_ht, is_comp_het=True, **kwargs) - self._comp_het_ht = self._filter_compound_hets() - - if families_ht is not None: - self._ht = self._query_table_annotations(families_ht, self._get_table_path('annotations.ht')) - self._ht = self._filter_annotated_table(self._ht, **kwargs) - - def _import_and_filter_entries_ht( - self, project_guid: str, num_families: int, project_sample_type_data, **kwargs - ) -> tuple[hl.Table, hl.Table]: - sample_type = list(project_sample_type_data.keys())[0] - ht, sample_data = self._load_family_or_project_ht( - num_families, project_guid, project_sample_type_data, sample_type, **kwargs - ) - return self._filter_single_entries_table(ht, sample_data, **kwargs) - - def _load_family_or_project_ht(self, num_families, project_guid, project_sample_type_data, sample_type, **kwargs): - if num_families == 1: - family_guid = list(project_sample_type_data[sample_type].keys())[0] - ht, sample_data = self._load_family_ht(family_guid, sample_type, project_sample_type_data, **kwargs) - else: - ht, sample_data = self._load_project_ht(project_guid, sample_type, project_sample_type_data, **kwargs) - return ht, sample_data - - def _load_family_ht( - self, family_guid: str, sample_type: str, project_sample_type_data: dict, **kwargs - ) -> tuple[hl.Table, dict]: - ht = self._read_table(f'families/{sample_type}/{family_guid}.ht') - ht = ht.transmute(family_entries=[ht.entries]) - ht = ht.annotate_globals(family_guids=[family_guid], family_samples={family_guid: ht.sample_ids}) - ht = self._prefilter_entries_table(ht, **kwargs) - sample_data = project_sample_type_data[sample_type] - return ht, sample_data - - def _load_project_ht( - self, project_guid: str, sample_type: str, project_sample_type_data: dict, **kwargs - ) -> tuple[hl.Table, dict]: - ht = self._read_project_table(project_guid, sample_type) - ht = self._prefilter_entries_table(ht, **kwargs) - sample_data = project_sample_type_data[sample_type] - return ht, sample_data - - def _read_project_table(self, project_guid: str, sample_type: str, **kwargs): - return self._read_table(f'projects/{sample_type}/{project_guid}.ht', **kwargs) - - def _read_project_data(self, project_guid: str, sample_type: str, **kwargs): - project_ht = self._read_project_table(project_guid, sample_type, **kwargs) - if project_ht is not None: - project_ht = project_ht.select_globals('sample_type', 'family_guids', 'family_samples') - return project_ht - - def _prefilter_merged_project_hts(self, project_hts, n_partitions, **kwargs): - ht = self._merge_project_hts(project_hts, n_partitions, include_all_globals=True) - return self._prefilter_entries_table(ht, **kwargs) - - @classmethod - def _merge_project_hts(cls, project_hts, n_partitions, include_all_globals=False): - if not project_hts: - return None - ht = hl.Table.multi_way_zip_join(project_hts, 'project_entries', 'project_globals') - ht = ht.repartition(n_partitions) - project_entries = ht.project_entries - if include_all_globals: - project_entries = project_entries.map(cls._annotate_entry_filters) - ht = ht.transmute( - family_entries=hl.enumerate(project_entries).starmap(lambda i, x: hl.or_else( - x.family_entries, - ht.project_globals[i].family_guids.map(lambda f: hl.missing(x.family_entries.dtype.element_type)), - )).flatmap(lambda x: x), - ) - global_expressions = { - 'family_guids': ht.project_globals.flatmap(lambda x: x.family_guids), - } - if include_all_globals: - global_expressions.update({ - 'sample_types': ht.project_globals.flatmap(lambda x: x.family_guids.map(lambda _: x.sample_type)), - 'family_samples': hl.dict(ht.project_globals.flatmap(lambda x: x.family_samples.items())), - }) - - return ht.transmute_globals(**global_expressions) - - def _merge_filtered_hts(self, filtered_comp_het_project_hts, filtered_project_hts, n_partitions): - ht = self._merge_project_hts(filtered_project_hts, n_partitions) - comp_het_ht = self._merge_project_hts(filtered_comp_het_project_hts, n_partitions) - return ht, comp_het_ht - - @staticmethod - def _apply_entry_filters(ht, select_globals=True): - if ht is not None: - ht = ht.filter(ht.family_entries.any(hl.is_defined)) - if select_globals: - ht = ht.select_globals('family_guids') - return ht - - def _filter_single_entries_table(self, ht, project_families, inheritance_filter=None, quality_filter=None, is_merged_ht=False, **kwargs): - ht, sorted_family_sample_data = self._add_entry_sample_families(ht, project_families, is_merged_ht) - ht = self._filter_quality(ht, quality_filter, **kwargs) - ht, ch_ht = self._filter_inheritance(ht, None, inheritance_filter, sorted_family_sample_data) - ht = self._apply_entry_filters(ht) - ch_ht = self._apply_entry_filters(ch_ht) - - return ht, ch_ht - - def _filter_quality( - self, ht, quality_filter, annotation='family_entries', entries_ht_field='family_entries', **kwargs - ): - passes_quality_filter = self._get_family_passes_quality_filter( - quality_filter, ht, **kwargs - ) - ht_entries = ht[entries_ht_field] - - if passes_quality_filter is not None: - return ht.annotate(**{ - annotation: ht_entries.map( - lambda entries: hl.or_missing(passes_quality_filter(entries), entries) - )}) - - return ht.annotate(**{annotation: ht_entries}) - - def _add_entry_sample_families(self, ht, sample_data, is_merged_ht): - """ - Annotates samples in family_entries with additional sample-level data. - returns a tuple containing: - - ht (hl.Table) - - sorted_family_sample_data (list): A list of lists containing sample data for each family, - sorted in the same order as is in family_entries. [[, ]] - """ - ht_globals = hl.eval(ht.globals) - - missing_samples = set() - family_sample_index_data = [] - sorted_family_sample_data = [] - family_guids = sorted(sample_data.keys()) - for family_guid in family_guids: - ht_family_samples = ht_globals.family_samples[family_guid] - samples = sample_data[family_guid] - if samples is True: - samples = ht_family_samples - get_sample_data = lambda s: {'sampleId': s} - missing_family_samples = [] - else: - get_sample_data = self._sample_entry_data - missing_family_samples = [s['sample_id'] for s in samples if s['sample_id'] not in ht_family_samples] - if missing_family_samples: - missing_samples.update(missing_family_samples) - else: - family_index = ht_globals.family_guids.index(family_guid) - family_entry_data = { - 'sampleType': self._get_sample_type(family_index, ht_globals), - 'familyGuid': family_guid, - } - formatted_samples = [{**family_entry_data, **get_sample_data(s)} for s in samples] - sample_index_data = [(ht_family_samples.index(s['sampleId']), hl.struct(**s)) for s in formatted_samples] - family_sample_index_data.append((family_index, sample_index_data)) - sorted_family_sample_data.append(formatted_samples) - self.entry_samples_by_family_guid[family_guid] = [s['sampleId'] for s in formatted_samples] - - if missing_samples: - raise HTTPBadRequest( - reason=f'The following samples are available in seqr but missing the loaded data: {", ".join(sorted(missing_samples))}' - ) - - family_sample_index_data = hl.array(family_sample_index_data) - - ht = ht.annotate_globals(family_guids=family_guids) - - ht = ht.transmute(family_entries=family_sample_index_data.map(lambda family_tuple: family_tuple[1].map( - lambda sample_tuple: ht.family_entries[family_tuple[0]][sample_tuple[0]].annotate(**sample_tuple[1]) - ))) - if not is_merged_ht: - ht = self._annotate_entry_filters(ht) - - return ht, sorted_family_sample_data - - @staticmethod - def _annotate_entry_filters(ht): - return ht.annotate(family_entries=ht.family_entries.map( - lambda entries: entries.map(lambda x: x.annotate(filters=ht.filters)) - )) - - @classmethod - def _sample_entry_data(cls, sample): - return dict( - sampleId=sample['sample_id'], - individualGuid=sample['individual_guid'], - affected_id=AFFECTED_ID_MAP.get(sample['affected']), - is_male=sample.get('is_male', False), - ) - - @classmethod - def _get_sample_type(cls, family_index, ht_globals): - if 'sample_types' in ht_globals: - return ht_globals.sample_types[family_index] - return ht_globals.sample_type - - def _filter_inheritance( - self, ht, comp_het_ht, inheritance_filter, sorted_family_sample_data, - annotation='family_entries', entries_ht_field='family_entries', **kwargs - ): - any_valid_entry = lambda x: self.GENOTYPE_QUERY_MAP[HAS_ALT](x.GT) - - is_any_affected = self._inheritance_mode == ANY_AFFECTED - if is_any_affected: - prev_any_valid_entry = any_valid_entry - any_valid_entry = lambda x: prev_any_valid_entry(x) & (x.affected_id == AFFECTED_ID) - - ht = ht.annotate(**{ - entries_ht_field: ht[entries_ht_field].map( - lambda entries: hl.or_missing(entries.any(any_valid_entry), entries) - )}) - - if self._has_comp_het_search: - comp_het_ht = self._annotate_families_inheritance( - comp_het_ht if comp_het_ht is not None else ht, COMPOUND_HET, inheritance_filter, - sorted_family_sample_data, annotation, entries_ht_field, **kwargs - ) - - if is_any_affected or not (inheritance_filter or self._inheritance_mode): - # No sample-specific inheritance filtering needed - sorted_family_sample_data = [] - - ht = None if self._inheritance_mode == COMPOUND_HET else self._annotate_families_inheritance( - ht, self._inheritance_mode, inheritance_filter, sorted_family_sample_data, - annotation, entries_ht_field, **kwargs - ) - - return ht, comp_het_ht - - def _annotate_families_inheritance( - self, ht, inheritance_mode, inheritance_filter, sorted_family_sample_data, - annotation, entries_ht_field, family_passes_inheritance_filter = None - ): - if not family_passes_inheritance_filter: - family_passes_inheritance_filter = self._get_family_passes_inheritance_filter - - entry_indices_by_gt = self._get_entry_indices_by_gt_map( - inheritance_filter, inheritance_mode, sorted_family_sample_data - ) - - for genotype, entry_indices in entry_indices_by_gt.items(): - if not entry_indices: - continue - entry_indices = hl.dict(entry_indices) - ht = ht.annotate(**{ - annotation: hl.enumerate(ht[entries_ht_field]).starmap( - lambda family_idx, family_samples: family_passes_inheritance_filter( - entry_indices, family_idx, genotype, family_samples, ht, annotation - ) - ) - }) - - return ht - - def _get_family_passes_inheritance_filter(self, entry_indices, family_idx, genotype, family_samples, *args): - return hl.or_missing( - ~entry_indices.contains(family_idx) | entry_indices[family_idx].all( - lambda sample_i: self.GENOTYPE_QUERY_MAP[genotype](family_samples[sample_i].GT) - ), family_samples) - - def _get_entry_indices_by_gt_map(self, inheritance_filter, inheritance_mode, sorted_family_sample_data): - individual_genotype_filter = (inheritance_filter or {}).get('genotype') - - # Create a mapping of genotypes to check against a list of samples for a family - entry_indices_by_gt = defaultdict(lambda: defaultdict(list)) - for family_index, samples in enumerate(sorted_family_sample_data): - for sample_index, s in enumerate(samples): - genotype = individual_genotype_filter.get(s['individualGuid']) \ - if individual_genotype_filter else INHERITANCE_FILTERS[inheritance_mode].get(s['affected_id']) - if inheritance_mode == X_LINKED_RECESSIVE and s['affected_id'] == UNAFFECTED_ID and s['is_male']: - genotype = REF_REF - if genotype == COMP_HET_ALT and self._override_comp_het_alt: - genotype = HAS_ALT - if genotype: - entry_indices_by_gt[genotype][family_index].append(sample_index) - - if inheritance_mode == COMPOUND_HET: - family_unaffected_counts = [ - len(i) for i in entry_indices_by_gt[INHERITANCE_FILTERS[COMPOUND_HET][UNAFFECTED_ID]].values() - ] - self.max_unaffected_samples = max(family_unaffected_counts) if family_unaffected_counts else 0 - - return entry_indices_by_gt - - def _get_family_passes_quality_filter(self, quality_filter, ht, **kwargs): - quality_filter = quality_filter or {} - - affected_only = quality_filter.get('affected_only') - passes_quality_filters = [] - for filter_k, value in quality_filter.items(): - genotype_key = filter_k.replace('min_', '') - field = self.GENOTYPE_QUERY_FIELDS.get(genotype_key, self.GENOTYPE_FIELDS.get(genotype_key)) - if field and value: - passes_quality_filters.append(self._get_genotype_passes_quality_field(field, value, affected_only)) - - if quality_filter.get('vcf_filter'): - passes_quality_filters.append(self._passes_vcf_filters) - - if not passes_quality_filters: - return None - - return lambda entries: entries.all(lambda gt: hl.all([f(gt) for f in passes_quality_filters])) - - @classmethod - def _get_genotype_passes_quality_field(cls, field, value, affected_only, **kwargs): - field_config = cls.QUALITY_FILTER_FORMAT.get(field) or QualityFilterFormat() - if field_config.scale: - value = value / field_config.scale - - def passes_quality_field(gt): - is_valid = (gt[field] >= value) | hl.is_missing(gt[field]) - if field_config.override: - is_valid |= field_config.override(gt) - if affected_only: - is_valid |= gt.affected_id != AFFECTED_ID - return is_valid - - return passes_quality_field - - @staticmethod - def _passes_vcf_filters(gt): - return hl.is_missing(gt.filters) | (gt.filters.length() < 1) - - def _parse_variant_keys(self, variant_keys): - return [hl.struct(**{self.KEY_FIELD[0]: key}) for key in (variant_keys or [])] - - def _prefilter_entries_table(self, ht, **kwargs): - return ht - - def _filter_annotated_table(self, ht, gene_ids=None, rs_ids=None, frequencies=None, in_silico=None, pathogenicity=None, - parsed_annotations=None, is_comp_het=False, exclude=None, **kwargs): - - ht = self._filter_excluded(ht, exclude) - - ht = self._filter_by_gene_ids(ht, hl.set(gene_ids) if gene_ids else None) - - if rs_ids: - ht = self._filter_rs_ids(ht, rs_ids) - - ht = self._filter_by_frequency(ht, frequencies, pathogenicity) - - ht = self._filter_by_in_silico(ht, in_silico) - - return self._filter_by_annotations(ht, is_comp_het=is_comp_het, **(parsed_annotations or {})) - - @classmethod - def _annotate_filtered_gene_transcripts(cls, ht, gene_ids): - return ht.annotate( - **{FILTERED_GENE_TRANSCRIPTS: ht[cls.TRANSCRIPTS_FIELD].filter(lambda t: gene_ids.contains(t.gene_id))} - ) - - def _filter_excluded(self, ht, exclude): - return ht - - def _filter_by_gene_ids(self, ht, gene_ids): - if gene_ids is None: - return ht - - ht = self._annotate_filtered_gene_transcripts(ht, gene_ids) - return ht.filter(hl.is_defined(ht[FILTERED_GENE_TRANSCRIPTS].first())) - - def _filter_rs_ids(self, ht, rs_ids): - rs_id_set = hl.set(rs_ids) - return ht.filter(rs_id_set.contains(ht.rsid)) - - def _parse_intervals(self, intervals, gene_ids=None, variant_keys=None, variant_ids=None, exclude_intervals=False, **kwargs): - parsed_variant_keys = self._parse_variant_keys(variant_keys) - if parsed_variant_keys: - self._load_table_kwargs['variant_ht'] = hl.Table.parallelize(parsed_variant_keys).key_by(*self.KEY_FIELD) - return intervals - - if variant_ids: - first_chrom = variant_ids[0][0] - if all(first_chrom == v[0] for v in variant_ids): - positions = [pos for _, pos, _, _ in variant_ids] - intervals = [(first_chrom, min(positions), max(positions) + 1)] - else: - intervals = [(chrom, pos, pos+1) for chrom, pos, _, _ in variant_ids] - - is_x_linked = self._inheritance_mode == X_LINKED_RECESSIVE - if not (intervals or is_x_linked): - return intervals - - raw_intervals = intervals - if self._should_add_chr_prefix(): - intervals = [[f'chr{interval[0]}', *interval[1:]] for interval in (intervals or [])] - - if len(intervals) > MAX_GENE_INTERVALS and len(intervals) == len(gene_ids or []): - intervals = self.cluster_intervals(sorted(intervals)) - - parsed_intervals = [ - hl.eval(hl.locus_interval(*interval, reference_genome=self.GENOME_VERSION, invalid_missing=True)) - for interval in (intervals or []) - ] - invalid_intervals = [raw_intervals[i] for i, interval in enumerate(parsed_intervals) if interval is None] - if invalid_intervals: - error_interval = ', '.join([f'{chrom}:{start}-{end}' for chrom, start, end in invalid_intervals]) - raise HTTPBadRequest(reason=f'Invalid intervals: {error_interval}') - - if is_x_linked: - reference_genome = hl.get_reference(self.GENOME_VERSION) - parsed_intervals.append( - hl.eval(hl.parse_locus_interval(reference_genome.x_contigs[0], reference_genome=self.GENOME_VERSION)) - ) - - if parsed_intervals and not exclude_intervals: - self._set_interval_prefilter(parsed_intervals) - - return parsed_intervals - - def _set_interval_prefilter(self, parsed_intervals): - if len(parsed_intervals) < MAX_LOAD_INTERVALS: - self._load_table_kwargs = {'_intervals': parsed_intervals, '_filter_intervals': True} - - @classmethod - def cluster_intervals(cls, intervals, distance=100000, max_intervals=MAX_GENE_INTERVALS): - if len(intervals) <= max_intervals: - return intervals - - merged_intervals = [intervals[0]] - for chrom, start, end in intervals[1:]: - prev_chrom, prev_start, prev_end = merged_intervals[-1] - if chrom == prev_chrom and start - prev_end < distance: - merged_intervals[-1] = [chrom, prev_start, max(prev_end, end)] - else: - merged_intervals.append([chrom, start, end]) - - return cls.cluster_intervals(merged_intervals, distance=distance+100000, max_intervals=max_intervals) - - def _should_add_chr_prefix(self): - return self.GENOME_VERSION == GENOME_VERSION_GRCh38 - - def _filter_by_frequency(self, ht, frequencies, pathogenicity): - frequencies = {k: v for k, v in (frequencies or {}).items() if k in self.POPULATIONS} - if not frequencies: - return ht - - path_override_filter = self._frequency_override_filter(ht, pathogenicity) - filters = [] - for pop, freqs in sorted(frequencies.items()): - pop_filters = [] - pop_expr = ht[self.POPULATION_FIELDS.get(pop, pop)] - pop_config = self._format_population_config(self.POPULATIONS[pop]) - if freqs.get('af') is not None and freqs['af'] < 1: - af_field = pop_config.get('filter_af') or pop_config['af'] - if af_field is None: - continue - pop_filter = pop_expr[af_field] <= freqs['af'] - if path_override_filter is not None and freqs['af'] < PATH_FREQ_OVERRIDE_CUTOFF: - pop_filter |= path_override_filter & (pop_expr[af_field] <= PATH_FREQ_OVERRIDE_CUTOFF) - pop_filters.append(pop_filter) - elif freqs.get('ac') is not None: - ac_field = pop_config['ac'] - if ac_field: - pop_filters.append(pop_expr[ac_field] <= freqs['ac']) - - if freqs.get('hh') is not None: - for field_name in ['hom', 'hemi']: - field = pop_config[field_name] - if field: - pop_filter = pop_expr[field] <= freqs['hh'] - if path_override_filter is not None: - pop_filter |= path_override_filter - pop_filters.append(pop_filter) - - if pop_filters: - filters.append(hl.is_missing(pop_expr) | hl.all(pop_filters)) - - if filters: - ht = ht.filter(hl.all(filters)) - return ht - - def _frequency_override_filter(self, ht, pathogenicity): - return None - - def _filter_by_in_silico(self, ht, in_silico_filters): - in_silico_filters = in_silico_filters or {} - require_score = in_silico_filters.get('requireScore', False) - in_silico_filters = {k: v for k, v in in_silico_filters.items() if k in self.PREDICTION_FIELDS_CONFIG and v} - if not in_silico_filters: - return ht - - in_silico_qs = [] - missing_qs = [] - for in_silico, value in in_silico_filters.items(): - score_filter, ht_value = self._get_in_silico_filter(ht, in_silico, value) - in_silico_qs.append(score_filter) - if not require_score: - missing_qs.append(hl.is_missing(ht_value)) - - if missing_qs: - in_silico_qs.append(hl.all(missing_qs)) - - return ht.filter(hl.any(in_silico_qs)) - - def _get_in_silico_filter(self, ht, in_silico, value): - score_path = self.PREDICTION_FIELDS_CONFIG[in_silico] - enum_lookup = self._get_enum_lookup(*score_path[:2]) - if enum_lookup is not None: - ht_value = ht[score_path.source][f'{score_path.field}_id'] - score_filter = ht_value == enum_lookup[value] - else: - ht_value = ht[score_path.source][score_path.field] - score_filter = ht_value >= float(value) - - return score_filter, ht_value - - def _parse_annotations(self, annotations, annotations_secondary, **kwargs): - annotations = annotations or {} - allowed_consequence_ids = self._get_allowed_consequence_ids(annotations) - annotation_overrides = self._get_annotation_override_fields(annotations, **kwargs) - - parsed_annotations = {} - if self._has_comp_het_search and annotations_secondary: - secondary_allowed_consequence_ids = self._get_allowed_consequence_ids(annotations_secondary) - has_different_secondary = secondary_allowed_consequence_ids != allowed_consequence_ids - has_data_type_primary_annotations = allowed_consequence_ids or annotation_overrides - has_data_type_secondary_annotations = bool(secondary_allowed_consequence_ids) - secondary_annotation_overrides = None - if self.SECONDARY_ANNOTATION_OVERRIDE_FIELDS: - secondary_annotation_overrides = self._get_annotation_override_fields( - annotations_secondary, override_fields=self.SECONDARY_ANNOTATION_OVERRIDE_FIELDS) - has_data_type_secondary_annotations |= bool(secondary_annotation_overrides) - has_different_secondary |= secondary_annotation_overrides != annotation_overrides - - if not has_data_type_primary_annotations: - allowed_consequence_ids = secondary_allowed_consequence_ids - annotation_overrides = secondary_annotation_overrides - # Data type only has annotations for second hit, so there is no need for the homozygous recessive query - self._inheritance_mode = COMPOUND_HET - elif has_different_secondary: - parsed_annotations.update({ - 'secondary_consequence_ids': secondary_allowed_consequence_ids, - 'secondary_annotation_overrides': secondary_annotation_overrides, - }) - self._has_secondary_annotations = True - - if not (has_data_type_primary_annotations and has_data_type_secondary_annotations): - self._is_multi_data_type_comp_het = True - - parsed_annotations.update({ - 'consequence_ids': allowed_consequence_ids, - 'annotation_overrides': annotation_overrides, - }) - return parsed_annotations - - def _filter_by_annotations(self, ht, is_comp_het=False, consequence_ids=None, annotation_overrides=None, - secondary_consequence_ids=None, secondary_annotation_overrides=None, **kwargs): - - annotation_exprs = {} - if consequence_ids: - annotation_exprs[ALLOWED_TRANSCRIPTS] = self._get_allowed_transcripts(ht, consequence_ids) - ht = ht.annotate(**annotation_exprs) - annotation_filters = self._get_annotation_override_filters(ht, annotation_overrides or {}) - - if not is_comp_het: - if annotation_exprs: - annotation_filters.append(self._has_allowed_transcript_filter(ht, ALLOWED_TRANSCRIPTS)) - if annotation_filters: - ht = ht.filter(hl.any(annotation_filters)) - return ht - - if annotation_filters: - annotation_exprs[HAS_ANNOTATION_OVERRIDE] = hl.any(annotation_filters) - if secondary_annotation_overrides is not None: - overrides = self._get_annotation_override_filters(ht, secondary_annotation_overrides) - annotation_exprs[f'{HAS_ANNOTATION_OVERRIDE}_secondary'] = hl.any(overrides) if overrides else False - secondary_allowed_transcripts_field = f'{ALLOWED_TRANSCRIPTS}_secondary' - if secondary_consequence_ids: - annotation_exprs[secondary_allowed_transcripts_field] = self._get_allowed_transcripts(ht, secondary_consequence_ids) - - filter_fields = list(annotation_exprs.keys()) - transcript_filter_fields = [ALLOWED_TRANSCRIPTS, secondary_allowed_transcripts_field] - annotation_exprs.pop(ALLOWED_TRANSCRIPTS, None) - if annotation_exprs: - ht = ht.annotate(**annotation_exprs) - - all_filters = [ - self._has_allowed_transcript_filter(ht, field) if field in transcript_filter_fields else ht[field] - for field in filter_fields - ] - return ht.filter(hl.any(all_filters)) - - def _get_allowed_consequence_ids(self, annotations): - allowed_consequences = { - ann for field, anns in annotations.items() - if anns and (field not in ANNOTATION_OVERRIDE_FIELDS) for ann in anns - } - return self._get_enum_terms_ids(self.TRANSCRIPTS_FIELD, self.TRANSCRIPT_CONSEQUENCE_FIELD, allowed_consequences) - - def _get_allowed_transcripts(self, ht, allowed_consequence_ids): - transcript_filter = self._get_allowed_transcripts_filter(allowed_consequence_ids) - return getattr(ht, FILTERED_GENE_TRANSCRIPTS, ht[self.TRANSCRIPTS_FIELD]).filter(transcript_filter) - - @staticmethod - def _get_allowed_transcripts_filter(allowed_consequence_ids): - allowed_consequence_ids = hl.set(allowed_consequence_ids) - return lambda gc: allowed_consequence_ids.contains(gc.major_consequence_id) - - def _get_annotation_override_fields(self, annotations, override_fields=None, **kwargs): - override_fields = override_fields or self.ANNOTATION_OVERRIDE_FIELDS - return {k: annotations[k] for k in override_fields if annotations.get(k)} - - def _get_annotation_override_filters(self, ht, annotation_overrides): - return [] - - def _get_annotation_filters(self, ht, is_secondary=False): - suffix = '_secondary' if is_secondary else '' - annotation_filters = [] - - allowed_transcripts_field = f'{ALLOWED_TRANSCRIPTS}{suffix}' - if allowed_transcripts_field in ht.row: - annotation_filters.append(self._has_allowed_transcript_filter(ht, allowed_transcripts_field)) - - annotation_override_field = f'{HAS_ANNOTATION_OVERRIDE}{suffix}' - if annotation_override_field in ht.row: - annotation_filters.append(ht[annotation_override_field]) - elif HAS_ANNOTATION_OVERRIDE in ht.row: - # For secondary annotations, if no secondary override is defined use the primary override - annotation_filters.append(ht[HAS_ANNOTATION_OVERRIDE]) - - return annotation_filters - - @staticmethod - def _has_allowed_transcript_filter(ht, allowed_transcript_field): - return hl.is_defined(ht[allowed_transcript_field].first()) - - def _filter_compound_hets(self): - # pylint: disable=pointless-string-statement - ch_ht = self._comp_het_ht - - # Get possible pairs of variants within the same gene - def key(v): - ks = [v[k] for k in self.KEY_FIELD] - return ks[0] if len(self.KEY_FIELD) == 1 else hl.tuple(ks) - ch_ht = ch_ht.annotate(key_=key(ch_ht.row), gene_ids=self._gene_ids_expr(ch_ht, filtered_genes_only=True)) - ch_ht = ch_ht.explode(ch_ht.gene_ids) - - # Filter allowed transcripts to the grouped gene - transcript_annotations = { - k: ch_ht[k].filter(lambda t: t.gene_id == ch_ht.gene_ids) - for k in [ALLOWED_TRANSCRIPTS, ALLOWED_SECONDARY_TRANSCRIPTS] if k in ch_ht.row - } - if transcript_annotations: - ch_ht = ch_ht.annotate(**transcript_annotations) - - if transcript_annotations or self._has_secondary_annotations: - primary_filters = self._get_annotation_filters(ch_ht) - ch_ht = ch_ht.annotate(is_primary=hl.coalesce(hl.any(primary_filters), False)) - if self._has_secondary_annotations and not self._is_multi_data_type_comp_het: - secondary_filters = self._get_annotation_filters(ch_ht, is_secondary=True) - is_secondary = hl.coalesce(hl.any(secondary_filters), False) - else: - is_secondary = ch_ht.is_primary - ch_ht = ch_ht.annotate(is_secondary=is_secondary) - if transcript_annotations: - ch_ht = ch_ht.filter(ch_ht.is_primary | ch_ht.is_secondary) - else: - ch_ht = ch_ht.annotate(is_primary=True, is_secondary=True) - self.unfiltered_comp_het_ht = ch_ht - - if self._is_multi_data_type_comp_het: - # In cases where comp het pairs must have different data types, there are no single data type results - return None - - """ Assume a table with the following data - key_ | gene_ids | is_primary | is_secondary - 1 | A | true | true - 2 | A | true | true - 2 | B | true | true - 3 | A | false | true - 3 | B | false | true - """ - - variants = ch_ht.collect(_localize=False) - variants = variants.group_by(lambda v: v.gene_ids) - variants = variants.items().flatmap(lambda gvs: - hl.rbind(gvs[0], gvs[1], lambda gene_id, variants: - hl.rbind( - variants.filter(lambda v: v.is_primary), - variants.filter(lambda v: v.is_secondary), - lambda v1s, v2s: - v1s.flatmap(lambda v1: - v2s \ - .filter(lambda v2: ~v2.is_primary | ~v1.is_secondary | (v1.key_ < v2.key_)) \ - .map(lambda v2: hl.tuple([gene_id, v1, v2])) - ) - ) - ) - ) - - """ After grouping by gene and pairing/filtering have array of tuples (gene_id, v1, v2) - (A, 1, 2) - (A, 1, 3) - (A, 2, 3) - (B, 2, 3) - """ - - variants = variants.group_by(lambda v: hl.tuple([v[1].key_, v[2].key_])) - variants = variants.values().map(lambda v: hl.rbind( - hl.set(v.map(lambda v: v[0])), - lambda comp_het_gene_ids: hl.struct( - v1=v[0][1].annotate(comp_het_gene_ids=comp_het_gene_ids), - v2=v[0][2].annotate(comp_het_gene_ids=comp_het_gene_ids), - ) - )) - - """ After grouping by pair have array of structs (v1, v2) annotated with comp_het_gene_ids - v1 | v2 | v<1/2>.comp_het_gene_ids - 1 | 2 | {A} - 1 | 3 | {A} - 2 | 3 | {A, B} - """ - - variants = self._filter_comp_het_families(variants) - - return hl.Table.parallelize( - variants.map(lambda v: hl.struct(**{GROUPED_VARIANTS_FIELD: hl.array([v.v1, v.v2])})) - ) - - def _filter_comp_het_families(self, variants, set_secondary_annotations=True): - variants = self._annotate_comp_het_valid_family_indices(variants) - return variants.filter( - lambda v: v.valid_family_indices.any(hl.is_defined) - ).map(lambda v: v.select( - v1=self._annotated_comp_het_variant(v.v1, v.valid_family_indices), - v2=self._annotated_comp_het_variant(v.v2, v.valid_family_indices, is_secondary=set_secondary_annotations), - )) - - def _annotate_comp_het_valid_family_indices(self, variants): - return variants.map(lambda v: v.annotate( - valid_family_indices=hl.enumerate(v.v1.family_entries).map(lambda x: x[0]).filter( - lambda i: self._is_valid_comp_het_family(v.v1, v.v2, i) - ) - )) - - def _annotated_comp_het_variant(self, variant, valid_family_indices, is_secondary=False): - if is_secondary and self._has_secondary_annotations and ALLOWED_TRANSCRIPTS in variant and ALLOWED_SECONDARY_TRANSCRIPTS in variant: - variant = variant.annotate(**{ALLOWED_TRANSCRIPTS: variant[ALLOWED_SECONDARY_TRANSCRIPTS]}) - - return variant.annotate( - family_entries=valid_family_indices.map(lambda i: variant.family_entries[i]), - ) - - @classmethod - def _gene_ids_expr(cls, ht, filtered_genes_only=False): - field = FILTERED_GENE_TRANSCRIPTS if filtered_genes_only and hasattr(ht, FILTERED_GENE_TRANSCRIPTS) else cls.TRANSCRIPTS_FIELD - return hl.set(ht[field].map(lambda t: t.gene_id)) - - def _is_valid_comp_het_family(self, v1, v2, family_index): - entries_1 = v1.family_entries[family_index] - entries_2 = v2.family_entries[family_index] - family_filters = [hl.is_defined(entries_1), hl.is_defined(entries_2)] - if self.max_unaffected_samples > 0: - family_filters.append(hl.enumerate(entries_1).all(lambda x: hl.any([ - (x[1].affected_id != UNAFFECTED_ID), *self._comp_het_entry_has_ref(x[1].GT, entries_2[x[0]].GT), - ]))) - if self._override_comp_het_alt: - family_filters.append(entries_1.extend(entries_2).all(lambda x: ~self.GENOTYPE_QUERY_MAP[ALT_ALT](x.GT))) - return hl.all(family_filters) - - def _comp_het_entry_has_ref(self, gt1, gt2): - return [self.GENOTYPE_QUERY_MAP[REF_REF](gt1), self.GENOTYPE_QUERY_MAP[REF_REF](gt2)] - - def _format_comp_het_results(self, ch_ht, annotation_fields): - formatted_grouped_variants = hl.array([ - self._format_results(ch_ht[GROUPED_VARIANTS_FIELD][0], annotation_fields=annotation_fields), - self._format_results(ch_ht[GROUPED_VARIANTS_FIELD][1], annotation_fields=annotation_fields), - ]) - ch_ht = ch_ht.annotate(**{GROUPED_VARIANTS_FIELD: hl.sorted(formatted_grouped_variants, key=lambda x: x._sort)}) - return ch_ht.annotate(_sort=ch_ht[GROUPED_VARIANTS_FIELD][0]._sort) - - def _format_results(self, ht, annotation_fields=None, **kwargs): - if annotation_fields is None: - annotation_fields = self.annotation_fields() - annotations = {k: v(ht) for k, v in annotation_fields.items()} - annotations.update({ - '_sort': self._sort_order(ht), - 'genomeVersion': self.GENOME_VERSION.replace('GRCh', ''), - }) - results = ht.annotate(**annotations) - return results.select(*self.CORE_FIELDS, *list(annotations.keys())) - - def format_search_ht(self): - ch_ht = None - annotation_fields = self.annotation_fields() - if self._comp_het_ht: - ch_ht = self._format_comp_het_results(self._comp_het_ht, annotation_fields) - - if self._ht: - ht = self._format_results(self._ht.key_by(), annotation_fields=annotation_fields) - if ch_ht: - ht = ht.union(ch_ht, unify=True) - else: - ht = ch_ht - return ht - - def search(self): - ht = self.format_search_ht() - - (total_results, collected) = (0, None) if ht is None else ht.aggregate((hl.agg.count(), hl.agg.take(ht.row, self._num_results, ordering=ht._sort))) - logger.info(f'Total hits: {total_results}. Fetched: {self._num_results}') - - return [] if collected is None else self._format_collected_rows(collected), total_results - - def _format_collected_rows(self, collected): - if self._has_comp_het_search: - return [row.get(GROUPED_VARIANTS_FIELD) or row.drop(GROUPED_VARIANTS_FIELD) for row in collected] - return collected - - def _sort_order(self, ht): - sort_expressions = self._get_sort_expressions(ht, XPOS) - if self._sort != XPOS: - sort_expressions = self._get_sort_expressions(ht, self._sort) + sort_expressions - return sort_expressions - - @staticmethod - def _format_prediction_sort_value(value): - return hl.or_else(-hl.float64(value), 0) - - def _get_sort_expressions(self, ht, sort): - if sort in self.SORTS: - return self.SORTS[sort](ht) - - if sort in self.PREDICTION_FIELDS_CONFIG: - prediction_path = self.PREDICTION_FIELDS_CONFIG[sort] - return [self._format_prediction_sort_value(ht[prediction_path.source][prediction_path.field])] - - if sort == OMIM_SORT: - return self._omim_sort(ht, hl.set(set(self._sort_metadata))) - - if self._sort_metadata: - return self._gene_rank_sort(ht, hl.dict(self._sort_metadata)) - - sort_field, sort_subfield = next( - ((field, config.get('sort_subfield', 'af')) for field, config in self.POPULATIONS.items() if config.get('sort') == sort), - (None, None), - ) - if sort_field: - return [hl.float64(self.population_expression(ht, sort_field)[sort_subfield])] - - return [] - - @classmethod - def _omim_sort(cls, r, omim_gene_set): - return [-cls._gene_ids_expr(r).intersection(omim_gene_set).size()] - - @classmethod - def _gene_rank_sort(cls, r, gene_ranks): - return [hl.min(cls._gene_ids_expr(r).map(gene_ranks.get))] - - @classmethod - def _gene_count_selects(cls): - return { - 'gene_ids': cls._gene_ids_expr, - 'families': cls.BASE_ANNOTATION_FIELDS[FAMILY_GUID_FIELD], - } - - def format_gene_count_hts(self): - hts = [] - selects = self._gene_count_selects() - if self._comp_het_ht: - ch_ht = self._comp_het_ht.explode(self._comp_het_ht[GROUPED_VARIANTS_FIELD]) - hts.append(ch_ht.select(**{k: v(ch_ht[GROUPED_VARIANTS_FIELD]) for k, v in selects.items()})) - if self._ht: - hts.append(self._ht.select(**{k: v(self._ht) for k, v in selects.items()})) - return hts - - def gene_counts(self): - hts = self.format_gene_count_hts() - ht = hts[0].key_by() - for sub_ht in hts[1:]: - ht = ht.union(sub_ht.key_by(), unify=True) - - ht = ht.explode('gene_ids').explode('families') - return ht.aggregate(hl.agg.group_by( - ht.gene_ids, hl.struct(total=hl.agg.count(), families=hl.agg.counter(ht.families)) - )) - - def _filter_variant_ids(self, ht, variant_ids): - return ht - - def lookup_variants(self, variant_ids, additional_annotations=None): - self._parse_intervals(intervals=None, variant_ids=variant_ids, variant_keys=variant_ids) - ht = self._read_table('annotations.ht', drop_globals=['versions']) - ht = self._filter_variant_ids(ht, variant_ids) - ht = ht.filter(hl.is_defined(ht[XPOS])) - - annotation_fields = { - k: v for k, v in self.annotation_fields(include_genotype_overrides=False).items() - if k not in {FAMILY_GUID_FIELD, GENOTYPES_FIELD} - } - if additional_annotations: - annotation_fields.update(additional_annotations) - formatted = self._format_results(ht.key_by(), annotation_fields=annotation_fields, include_genotype_overrides=False) - - return formatted.aggregate(hl.agg.take(formatted.row, len(variant_ids))) - - def _import_variant_projects_ht(self, variant_id, project_samples=None, **kwargs): - projects_ht, _ = self._import_and_filter_multiple_project_hts(project_samples, n_partitions=1) - return self._filter_variant_ids(projects_ht, [variant_id]).key_by() - - def _get_variant_project_data(self, variant_id, **kwargs): - projects_ht = self._import_variant_projects_ht(variant_id, **kwargs) - project_data = projects_ht.aggregate(hl.agg.take(projects_ht.row, 1)) - return project_data[0] if project_data else {} - - def lookup_variant(self, variant_id, **kwargs): - variants = self.lookup_variants([variant_id], additional_annotations=self._lookup_variant_annotations()) - if not variants: - raise HTTPNotFound() - variant = dict(variants[0]) - variant.update(self._get_variant_project_data(variant_id, variant=variant, **kwargs)) - return variant - - @staticmethod - def _lookup_variant_annotations(): - return {} diff --git a/hail_search/queries/gcnv.py b/hail_search/queries/gcnv.py deleted file mode 100644 index beae119b4b..0000000000 --- a/hail_search/queries/gcnv.py +++ /dev/null @@ -1,134 +0,0 @@ -import hail as hl - -from hail_search.constants import COMP_HET_ALT, HAS_ALT, HAS_REF, REF_REF, FILTERED_GENE_TRANSCRIPTS, \ - ALLOWED_TRANSCRIPTS, ALLOWED_SECONDARY_TRANSCRIPTS -from hail_search.queries.sv import SvHailTableQuery - - -class GcnvHailTableQuery(SvHailTableQuery): - - DATA_TYPE = 'SV_WES' - SV_TYPE_PREFIX = 'gCNV_' - - # gCNV data has no ref/ref calls so a missing entry indicates ref/ref - GENOTYPE_QUERY_MAP = { - **SvHailTableQuery.GENOTYPE_QUERY_MAP, - REF_REF: hl.is_missing, - HAS_REF: lambda gt: hl.is_missing(gt) | gt.is_het_ref(), - HAS_ALT: hl.is_defined, - COMP_HET_ALT: hl.is_defined, - } - MISSING_NUM_ALT = 0 - - GENOTYPE_FIELDS = { - **SvHailTableQuery.GENOTYPE_FIELDS, - **{f.lower(): f for f in ['QS', 'defragged']}, - } - del GENOTYPE_FIELDS['gq'] - GENOTYPE_QUERY_FIELDS = {} - GENOTYPE_OVERRIDE_FIELDS = { - 'start': (hl.min, lambda r: r.start_locus.position), - 'end': (hl.max, lambda r: r.end_locus.position), - 'num_exon': (hl.max, lambda r: r.num_exon), - 'gene_ids': ( - lambda entry_gene_ids: entry_gene_ids.fold(lambda s1, s2: s1.union(s2), hl.empty_set(hl.tstr)), - lambda r: hl.missing(hl.tset(hl.tstr)), - ), - } - COMPUTED_GENOTYPE_FIELDS = { - **SvHailTableQuery.COMPUTED_GENOTYPE_FIELDS, - **{k: lambda entry, field, r: hl.or_missing( - hl.is_missing(r[field]) | (r[field] != entry[f'sample_{field}']), entry[f'sample_{field}'] - ) for k in GENOTYPE_OVERRIDE_FIELDS.keys()}, - } - COMPUTED_GENOTYPE_FIELDS['prev_overlap'] = COMPUTED_GENOTYPE_FIELDS.pop('prev_num_alt') - - CORE_FIELDS = SvHailTableQuery.CORE_FIELDS[:-1] - BASE_ANNOTATION_FIELDS = { - **SvHailTableQuery.BASE_ANNOTATION_FIELDS, - 'pos': lambda r: r.start, - 'end': lambda r: r.end, - 'numExon': lambda r: r.num_exon, - } - del BASE_ANNOTATION_FIELDS['bothsidesSupport'] - - TRANSCRIPTS_ENUM_FIELD = SvHailTableQuery.ENUM_ANNOTATION_FIELDS[SvHailTableQuery.TRANSCRIPTS_FIELD] - ENUM_ANNOTATION_FIELDS = {SvHailTableQuery.TRANSCRIPTS_FIELD: { - **TRANSCRIPTS_ENUM_FIELD, - 'format_array_values': lambda values, r: GcnvHailTableQuery.TRANSCRIPTS_ENUM_FIELD['format_array_values']( - hl.if_else(hl.is_missing(r.gene_ids), values, values.filter(lambda t: r.gene_ids.contains(t.geneId))), r, - ), - }} - - POPULATIONS = {k: v for k, v in SvHailTableQuery.POPULATIONS.items() if k != 'gnomad_svs'} - - @classmethod - def _get_genotype_override_field(cls, r, field): - agg, get_default = cls.GENOTYPE_OVERRIDE_FIELDS[field] - sample_field = f'sample_{field}' - entries = r.family_entries.flatmap(lambda x: x) - return hl.if_else( - entries.any(lambda g: hl.is_defined(g.GT) & hl.is_missing(g[sample_field])), - get_default(r), agg(entries.map(lambda g: g[sample_field])) - ) - - def _format_results(self, ht, *args, include_genotype_overrides=True, **kwargs): - ht = ht.annotate(**{ - k: self._get_genotype_override_field(ht, k) if include_genotype_overrides - else self.GENOTYPE_OVERRIDE_FIELDS[k][1](ht) - for k in self.GENOTYPE_OVERRIDE_FIELDS - }) - return super()._format_results(ht, *args, **kwargs) - - def get_allowed_sv_type_ids(self, sv_types): - return super().get_allowed_sv_type_ids([ - type.replace(self.SV_TYPE_PREFIX, '') for type in sv_types if type.startswith(self.SV_TYPE_PREFIX) - ]) - - def _annotate_families_table_annotations(self, families_ht, annotations_ht, is_comp_het=False): - ht = super()._annotate_families_table_annotations(families_ht, annotations_ht) - transcript_fields = [ - transcript_field for transcript_field in [ - ALLOWED_TRANSCRIPTS, ALLOWED_SECONDARY_TRANSCRIPTS, FILTERED_GENE_TRANSCRIPTS, - ] if hasattr(ht, transcript_field) - ] - if not transcript_fields: - return ht - - ht = ht.annotate(all_gene_ids=self._get_genotype_override_field(ht, 'gene_ids')) - for transcript_field in transcript_fields: - transcript_has_gene = lambda t: hl.is_missing(ht.all_gene_ids) | ht.all_gene_ids.contains(t.gene_id) - if transcript_field == FILTERED_GENE_TRANSCRIPTS: - ht = ht.filter(ht[transcript_field].any(transcript_has_gene)) - else: - ht = ht.annotate(**{transcript_field: ht[transcript_field].filter(transcript_has_gene)}) - if not is_comp_het: - # Will trigger re-filtering the ht for allowed annotations - self._has_secondary_annotations = True - - return ht.drop('all_gene_ids') - - def _filter_by_gene_ids(self, ht, gene_ids): - if not hasattr(ht, 'family_entries'): - return super()._filter_by_gene_ids(ht, gene_ids) - - # Gene and annotation filters should only consider genes present in the genotype override gene_ids - row_gene_ids = self._gene_ids_expr(ht) - - if gene_ids is None: - return self._annotate_filtered_gene_transcripts(ht, row_gene_ids) - - return super()._filter_by_gene_ids(ht, gene_ids.intersection(row_gene_ids)) - - @classmethod - def _gene_ids_expr(cls, ht, filtered_genes_only=False): - gene_ids_expr = super()._gene_ids_expr(ht) - if filtered_genes_only: - return gene_ids_expr - return hl.or_else( - cls._get_genotype_override_field(ht, 'gene_ids'), - gene_ids_expr, - ) - - def _additional_annotation_fields(self): - return {} diff --git a/hail_search/queries/mito.py b/hail_search/queries/mito.py deleted file mode 100644 index 59efeeaaef..0000000000 --- a/hail_search/queries/mito.py +++ /dev/null @@ -1,558 +0,0 @@ -from collections import defaultdict - -from aiohttp.web import HTTPNotFound -import hail as hl -import logging -import os - -from hail_search.constants import ABSENT_PATH_SORT_OFFSET, CLINVAR_KEY, CLINVAR_LIKELY_PATH_FILTER, \ - CLINVAR_PATH_FILTER, \ - CLINVAR_PATH_RANGES, CLINVAR_PATH_SIGNIFICANCES, ALLOWED_TRANSCRIPTS, ALLOWED_SECONDARY_TRANSCRIPTS, \ - PATHOGENICTY_SORT_KEY, CONSEQUENCE_SORT, FILTERED_GENE_TRANSCRIPTS, \ - PATHOGENICTY_HGMD_SORT_KEY, MAX_LOAD_INTERVALS -from hail_search.definitions import SampleType -from hail_search.queries.base import BaseHailTableQuery, PredictionPath, QualityFilterFormat, MAX_PARTITIONS - -REFERENCE_DATASETS_DIR = os.environ.get('REFERENCE_DATASETS_DIR', '/seqr/seqr-reference-data') - -logger = logging.getLogger(__name__) - - -def _clinvar_sort(clinvar_field, r): - return hl.or_else(r[clinvar_field].pathogenicity_id, ABSENT_PATH_SORT_OFFSET) - - -class MitoHailTableQuery(BaseHailTableQuery): - - DATA_TYPE = 'MITO' - KEY_FIELD = ('locus', 'alleles') - - TRANSCRIPTS_FIELD = 'sorted_transcript_consequences' - TRANSCRIPT_CONSEQUENCE_FIELD = 'consequence_term' - GENOTYPE_FIELDS = { - 'dp': 'DP', - 'hl': 'HL', - 'mitoCn': 'mito_cn', - 'contamination': 'contamination', - } - QUALITY_FILTER_FORMAT = { - 'HL': QualityFilterFormat(scale=100), - } - - POPULATION_FIELDS = {'helix': 'helix_mito', 'seqr': 'gt_stats'} - POPULATIONS = {} - for pop, sort in {'seqr': 'callset_af', 'gnomad_mito': 'gnomad', 'helix': None}.items(): - pop_het = f'{pop}_heteroplasmy' - POPULATIONS.update({ - pop: {'af': 'AF_hom', 'ac': 'AC_hom', 'hom': None, 'hemi': None, 'het': None, 'sort': sort}, - pop_het: { - 'af': 'AF_het', 'ac': 'AC_het', 'max_hl': None if pop == 'seqr' else 'max_hl', - 'hom': None, 'hemi': None, 'het': None, - }, - }) - POPULATION_FIELDS[pop_het] = POPULATION_FIELDS.get(pop, pop) - POPULATIONS['seqr']['sort_subfield'] = 'ac' - PREDICTION_FIELDS_CONFIG = { - 'apogee': PredictionPath('mitimpact', 'score'), - 'haplogroup_defining': PredictionPath('haplogroup', 'is_defining', lambda v: hl.or_missing(v, 'Y')), - 'hmtvar': PredictionPath('hmtvar', 'score'), - 'mitotip': PredictionPath('mitotip', 'trna_prediction'), - 'mut_taster': PredictionPath('dbnsfp', 'MutationTaster_pred'), - 'sift': PredictionPath('dbnsfp', 'SIFT_score'), - 'mlc': PredictionPath('local_constraint_mito', 'score'), - } - - PATHOGENICITY_FILTERS = { - CLINVAR_KEY: ('pathogenicity', CLINVAR_PATH_RANGES), - } - - GLOBALS = BaseHailTableQuery.GLOBALS + ['versions'] - CORE_FIELDS = BaseHailTableQuery.CORE_FIELDS + ['rsid'] - MITO_ANNOTATION_FIELDS = { - 'commonLowHeteroplasmy': lambda r: r.common_low_heteroplasmy, - 'mitomapPathogenic': lambda r: r.mitomap.pathogenic, - } - BASE_ANNOTATION_FIELDS = { - 'chrom': lambda r: r.locus.contig.replace("^chr", ""), - 'pos': lambda r: r.locus.position, - 'ref': lambda r: r.alleles[0], - 'alt': lambda r: r.alleles[1], - 'mainTranscriptId': lambda r: r.sorted_transcript_consequences.first().transcript_id, - 'selectedMainTranscriptId': lambda r: hl.or_missing( - r.selected_transcript != r.sorted_transcript_consequences.first(), r.selected_transcript.transcript_id, - ), - **MITO_ANNOTATION_FIELDS, - **BaseHailTableQuery.BASE_ANNOTATION_FIELDS, - } - ENUM_ANNOTATION_FIELDS = { - CLINVAR_KEY: { - 'response_key': CLINVAR_KEY, - 'include_version': True, - 'annotate_value': lambda value, enum: { - 'conflictingPathogenicities': MitoHailTableQuery._format_enum( - value, 'conflictingPathogenicities', enum, enum_keys=['pathogenicity']), - }, - }, - TRANSCRIPTS_FIELD: { - **BaseHailTableQuery.ENUM_ANNOTATION_FIELDS['transcripts'], - 'annotate_value': lambda transcript, *args: {'major_consequence': transcript.consequence_terms.first()}, - 'format_array_values': lambda values, *args: BaseHailTableQuery.ENUM_ANNOTATION_FIELDS['transcripts']['format_array_values'](values).map_values( - lambda transcripts: hl.enumerate(transcripts).starmap(lambda i, t: t.annotate(transcriptRank=i)) - ), - } - } - - CLINVAR_SORT = _clinvar_sort - SORTS = { - CONSEQUENCE_SORT: lambda r: [ - hl.min(r.sorted_transcript_consequences.flatmap(lambda t: t.consequence_term_ids)), - hl.min(r.selected_transcript.consequence_term_ids), - ], - PATHOGENICTY_SORT_KEY: lambda r: [_clinvar_sort(CLINVAR_KEY, r)], - **BaseHailTableQuery.SORTS, - } - SORTS[PATHOGENICTY_HGMD_SORT_KEY] = SORTS[PATHOGENICTY_SORT_KEY] - - PREFILTER_TABLES = { - CLINVAR_KEY: 'clinvar_path_variants.ht', - } - - def _import_and_filter_entries_ht( - self, project_guid: str, num_families: int, project_sample_type_data, **kwargs - ) -> tuple[hl.Table, hl.Table]: - sample_types = set(project_sample_type_data.keys()) - if len(sample_types) == 1: - # Import and filter entry table for one family or one project with one sample type - return super()._import_and_filter_entries_ht(project_guid, num_families, project_sample_type_data, **kwargs) - - self._has_both_sample_types = True - entries = {} - for sample_type in sample_types: - entries[sample_type] = self._load_family_or_project_ht( - num_families, project_guid, project_sample_type_data, sample_type, **kwargs - ) - - return self._filter_entries_ht_both_sample_types( - *entries[SampleType.WES.value], *entries[SampleType.WGS.value], **kwargs - ) - - def _import_and_filter_multiple_project_hts( - self, project_samples: dict, n_partitions=MAX_PARTITIONS, **kwargs - ) -> tuple[hl.Table, hl.Table]: - single_sample_type_project_samples = {} - both_sample_type_project_samples = defaultdict(dict) - for project_guid, sample_dict in project_samples.items(): - if len(sample_dict) == 1: - single_sample_type_project_samples[project_guid] = sample_dict - else: - both_sample_type_project_samples[SampleType.WES][project_guid] = {SampleType.WES: sample_dict[SampleType.WES]} - both_sample_type_project_samples[SampleType.WGS][project_guid] = {SampleType.WGS: sample_dict[SampleType.WGS]} - - filtered_project_hts = [] - filtered_comp_het_project_hts = [] - - # Process projects with only one sample type separately - if single_sample_type_project_samples: - ht, ch_ht = super()._import_and_filter_multiple_project_hts(single_sample_type_project_samples, n_partitions, **kwargs) - if ht is not None: - filtered_project_hts.append(ht) - if ch_ht is not None: - filtered_comp_het_project_hts.append(ch_ht) - - if both_sample_type_project_samples: - self._has_both_sample_types = True - wes_entries = self._load_project_hts( - both_sample_type_project_samples[SampleType.WES], n_partitions, **kwargs, - ) - wgs_entries = self._load_project_hts( - both_sample_type_project_samples[SampleType.WGS], n_partitions, **kwargs, - ) - for i, (wes_ht, wes_project_samples) in enumerate(wes_entries): - wgs_ht, wgs_project_samples = wgs_entries[i] - ht, comp_het_ht = self._filter_entries_ht_both_sample_types( - wes_ht, wes_project_samples, wgs_ht, wgs_project_samples, - is_merged_ht=True, **kwargs - ) - if ht is not None: - filtered_project_hts.append(ht) - if comp_het_ht is not None: - filtered_comp_het_project_hts.append(comp_het_ht) - - return self._merge_filtered_hts(filtered_comp_het_project_hts, filtered_project_hts, n_partitions) - - def _filter_entries_ht_both_sample_types( - self, wes_ht, wes_project_samples, wgs_ht, wgs_project_samples, inheritance_filter=None, quality_filter=None, - is_merged_ht=False, **kwargs - ): - wes_ht, sorted_wes_family_sample_data = self._add_entry_sample_families(wes_ht, wes_project_samples, is_merged_ht) - wgs_ht, sorted_wgs_family_sample_data = self._add_entry_sample_families(wgs_ht, wgs_project_samples, is_merged_ht) - wes_ht = wes_ht.rename({'family_entries': SampleType.WES.family_entries_field}) - wgs_ht = wgs_ht.rename({'family_entries': SampleType.WGS.family_entries_field}) - ht = wes_ht.join(wgs_ht, how='outer') - - sample_types = [ - (SampleType.WES, sorted_wes_family_sample_data), - (SampleType.WGS, sorted_wgs_family_sample_data) - ] - for sample_type, _ in sample_types: - ht = self._filter_quality( - ht, quality_filter, - annotation=sample_type.passes_quality_field, entries_ht_field=sample_type.family_entries_field, **kwargs - ) - - ch_ht = None - family_idx_map = defaultdict(dict) - for sample_type, sorted_family_sample_data in sample_types: - ht = self._annotate_initial_passes_inheritance(ht, sample_type) - ch_ht = self._annotate_initial_passes_inheritance(ch_ht, sample_type) - ht, ch_ht = self._filter_inheritance( - ht, ch_ht, inheritance_filter, sorted_family_sample_data, - annotation=sample_type.passes_inheritance_field, entries_ht_field=sample_type.family_entries_field, - family_passes_inheritance_filter=self._get_family_passes_inheritance_filter_both_sample_types - ) - for family_idx, samples in enumerate(sorted_family_sample_data): - family_guid = samples[0]['familyGuid'] - family_idx_map[family_guid][sample_type.value] = family_idx - - family_idx_map = hl.dict(family_idx_map) - ht = self._apply_multi_sample_type_entry_filters(ht, family_idx_map) - ch_ht = self._apply_multi_sample_type_entry_filters(ch_ht, family_idx_map) - return ht, ch_ht - - @staticmethod - def _annotate_initial_passes_inheritance(ht, sample_type): - if ht is None: - return ht - - return ht.annotate(**{ - sample_type.passes_inheritance_field: ht[sample_type.family_entries_field].map( - lambda family_entries: hl.array( - hl.range(0, hl.len(family_entries)).map(lambda _: True) - ) - )}) - - def _get_family_passes_inheritance_filter_both_sample_types( - self, entry_indices, family_idx, genotype, family_samples, ht, annotation - ): - return hl.enumerate(ht[annotation][family_idx]).starmap( - lambda sample_idx, passes: (hl.case() - .when(~entry_indices.get(family_idx).contains(sample_idx), passes) - .when(~self.GENOTYPE_QUERY_MAP[genotype](family_samples[sample_idx].GT), False) - .default(passes)) - ) - - def _apply_multi_sample_type_entry_filters(self, ht, family_idx_map): - if ht is None: - return ht - - for sample_type in SampleType: - ht = ht.annotate(**{ - sample_type.family_entries_field: hl.enumerate(ht[sample_type.family_entries_field]).starmap( - lambda family_idx, family_samples: hl.or_missing( - hl.bind(lambda other_sample_type_family_idx: (( - self._family_has_valid_quality(ht, sample_type, family_idx) | - self._family_has_valid_quality(ht, sample_type.other_sample_type, other_sample_type_family_idx) - ) & - self._family_has_valid_inheritance(ht, sample_type, family_idx, other_sample_type_family_idx) & - self._family_has_valid_inheritance(ht, sample_type.other_sample_type, other_sample_type_family_idx, family_idx) - ), family_idx_map.get(hl.coalesce(family_samples)[0]['familyGuid']).get(sample_type.other_sample_type.value), - ), family_samples) - )}) - - # Merge family entries and filters from both sample types - ht = ht.transmute( - family_entries=hl.coalesce( - ht[SampleType.WES.family_entries_field], hl.empty_array(ht[SampleType.WES.family_entries_field].dtype.element_type) - ).extend(hl.coalesce( - ht[SampleType.WGS.family_entries_field], hl.empty_array(ht[SampleType.WGS.family_entries_field].dtype.element_type) - )).filter(lambda entries: entries.any(hl.is_defined)) - ) - ht = ht.select('family_entries') - ht = ht.select_globals('family_guids') - - # Filter out families with no valid entries in either sample type - return ht.filter(ht.family_entries.any(hl.is_defined)) - - @staticmethod - def _family_has_valid_quality(ht, sample_type, sample_type_family_idx): - return ( - hl.is_defined(sample_type_family_idx) & - hl.is_defined(ht[sample_type.passes_quality_field][sample_type_family_idx]) - ) - - @staticmethod - def _family_has_valid_inheritance(ht, sample_type, family_idx, other_sample_type_family_idx): - return hl.bind( - lambda sample_type_fail_samples, other_sample_type_pass_samples: ( - sample_type_fail_samples.all(other_sample_type_pass_samples.contains) - ), hl.enumerate(ht[sample_type.family_entries_field][family_idx]).starmap( - lambda sample_idx, sample: hl.or_missing( - ~ht[sample_type.passes_inheritance_field][family_idx][sample_idx], - sample['sampleId'], - ) - ).filter(hl.is_defined), - hl.enumerate(ht[sample_type.other_sample_type.family_entries_field][other_sample_type_family_idx]).starmap( - lambda sample_idx, sample: hl.or_missing( - ht[sample_type.other_sample_type.passes_inheritance_field][other_sample_type_family_idx][sample_idx], - sample['sampleId'], - ) - ).filter(hl.is_defined), - ) - - def _annotate_comp_het_valid_family_indices(self, variants): - variants = super()._annotate_comp_het_valid_family_indices(variants) - if not self._has_both_sample_types: - return variants - - # Get valid comp het familyGuids - variants = variants.map(lambda v: v.annotate( - valid_family_guids=v.valid_family_indices.map( - lambda i: v.v1.family_entries[i][0]['familyGuid'] - ) - )) - # Re-annotate valid_family_indices based on valid familyGuids - return variants.map(lambda v: v.annotate( - valid_family_indices=hl.enumerate(v.v1.family_entries).filter( - lambda x: v.valid_family_guids.contains(x[1][0]['familyGuid']) - ).map(lambda x: x[0]) - )) - - - def _get_sample_genotype(self, samples, r=None, include_genotype_overrides=False, select_fields=None, **kwargs): - if not self._has_both_sample_types: - return super()._get_sample_genotype(samples, r, include_genotype_overrides, select_fields) - - return samples.map(lambda sample: self._select_genotype_for_sample( - sample, r, include_genotype_overrides, select_fields - )) - - @staticmethod - def _selected_main_transcript_expr(ht): - comp_het_gene_ids = getattr(ht, 'comp_het_gene_ids', None) - if comp_het_gene_ids is not None: - return hl.coalesce(*[ - ht[field].find(lambda t: comp_het_gene_ids.contains(t.gene_id)) for field in [ - ALLOWED_TRANSCRIPTS, ALLOWED_SECONDARY_TRANSCRIPTS, 'sorted_transcript_consequences', - ] if hasattr(ht, field) - ]) - - main_transcript = getattr(ht, FILTERED_GENE_TRANSCRIPTS, ht.sorted_transcript_consequences).first() - if hasattr(ht, ALLOWED_TRANSCRIPTS): - return hl.or_else(ht[ALLOWED_TRANSCRIPTS].first(), main_transcript) - return main_transcript - - def __init__(self, *args, **kwargs): - self._filter_hts = {} - self._has_both_sample_types = False - super().__init__(*args, **kwargs) - - def _get_family_passes_quality_filter(self, quality_filter, ht, pathogenicity=None, **kwargs): - passes_quality = super()._get_family_passes_quality_filter(quality_filter, ht) - clinvar_path_ht = False if passes_quality is None else self._get_loaded_clinvar_prefilter_ht(pathogenicity) - if not clinvar_path_ht: - return passes_quality - - return lambda entries: hl.is_defined(clinvar_path_ht[ht.key]) | passes_quality(entries) - - def _get_loaded_filter_ht(self, key, get_filters, **kwargs): - if self._filter_hts.get(key) is None: - ht_filter = get_filters(**kwargs) - if ht_filter is False: - self._filter_hts[key] = False - else: - ht = self._read_table(f'{self.PREFILTER_TABLES[key]}') - if ht_filter is not True: - ht = ht.filter(ht_filter(ht)) - self._filter_hts[key] = ht - - return self._filter_hts[key] - - @classmethod - def _get_table_dir(cls, path): - if any(prefilter_table_path in path for prefilter_table_path in cls.PREFILTER_TABLES.values()): - return REFERENCE_DATASETS_DIR - return super()._get_table_dir(path) - - def _get_loaded_clinvar_prefilter_ht(self, pathogenicity): - return self._get_loaded_filter_ht( - CLINVAR_KEY, self._get_clinvar_prefilter, pathogenicity=pathogenicity) - - def _get_clinvar_prefilter(self, pathogenicity=None): - clinvar_path_filters = self._get_clinvar_path_filters(pathogenicity) - if not clinvar_path_filters: - return False - - if CLINVAR_LIKELY_PATH_FILTER not in clinvar_path_filters: - return lambda ht: ht.is_pathogenic - elif CLINVAR_PATH_FILTER not in clinvar_path_filters: - return lambda ht: ht.is_likely_pathogenic - return True - - def _filter_variant_ids(self, ht, variant_ids): - if len(variant_ids) == 1: - variant_id_q = ht.alleles == [variant_ids[0][2], variant_ids[0][3]] - else: - variant_id_q = hl.any([ - (ht.locus == hl.locus(f'chr{chrom}' if self._should_add_chr_prefix() else chrom, pos, reference_genome=self.GENOME_VERSION)) & - (ht.alleles == [ref, alt]) - for chrom, pos, ref, alt in variant_ids - ]) - return ht.filter(variant_id_q) - - def _parse_variant_keys(self, variant_keys): - return None - - def _prefilter_entries_table(self, ht, parsed_intervals=None, exclude_intervals=False, variant_ids=None, **kwargs): - num_intervals = len(parsed_intervals or []) - if exclude_intervals and parsed_intervals: - ht = hl.filter_intervals(ht, parsed_intervals, keep=False) - elif num_intervals >= MAX_LOAD_INTERVALS: - ht = hl.filter_intervals(ht, parsed_intervals) - - if variant_ids: - ht = self._filter_variant_ids(ht, variant_ids) - - if '_n_partitions' not in self._load_table_kwargs and num_intervals > self._n_partitions: - ht = ht.naive_coalesce(self._n_partitions) - - return ht - - def _get_allowed_consequence_ids(self, annotations): - consequence_ids = super()._get_allowed_consequence_ids(annotations) - canonical_consequences = { - c.replace('__canonical', '') for consequences in annotations.values() - if consequences for c in consequences if c.endswith('__canonical') - } - if canonical_consequences: - canonical_consequence_ids = super()._get_allowed_consequence_ids({'canonical': canonical_consequences}) - canonical_consequence_ids -= consequence_ids - consequence_ids.update({f'{cid}__canonical' for cid in canonical_consequence_ids}) - return consequence_ids - - @staticmethod - def _get_allowed_transcripts_filter(allowed_consequence_ids): - canonical_consequences = set() - any_consequences = set() - for c in allowed_consequence_ids: - if str(c).endswith('__canonical'): - canonical_consequences.add(int(c.replace('__canonical', ''))) - else: - any_consequences.add(c) - - all_consequence_ids = None - if canonical_consequences: - all_consequence_ids = hl.set({*canonical_consequences, *any_consequences}) - - allowed_consequence_ids = hl.set(any_consequences) if any_consequences else hl.empty_set(hl.tint) - return lambda tc: tc.consequence_term_ids.any( - (hl.if_else(hl.is_defined(tc.canonical), all_consequence_ids, allowed_consequence_ids) - if canonical_consequences else allowed_consequence_ids - ).contains) - - def _filter_excluded(self, ht, exclude): - for key in self.PATHOGENICITY_FILTERS.keys(): - path_terms = (exclude or {}).get(key) - if path_terms: - ht = ht.filter(hl.is_missing(ht[key]) | ~self._has_path_expr(ht, path_terms, key)) - return ht - - def _get_annotation_override_fields(self, annotations, pathogenicity=None, **kwargs): - annotation_overrides = super()._get_annotation_override_fields(annotations, **kwargs) - for key in self.PATHOGENICITY_FILTERS.keys(): - path_terms = (pathogenicity or {}).get(key) - if path_terms: - annotation_overrides[key] = path_terms - return annotation_overrides - - def _get_annotation_override_filters(self, ht, annotation_overrides): - annotation_filters = [] - - for key in self.PATHOGENICITY_FILTERS.keys(): - if key in annotation_overrides: - annotation_filters.append(self._has_path_expr(ht, annotation_overrides[key], key)) - - return annotation_filters - - def _frequency_override_filter(self, ht, pathogenicity): - path_terms = self._get_clinvar_path_filters(pathogenicity) - return self._has_path_expr(ht, path_terms, CLINVAR_KEY) if path_terms else None - - @staticmethod - def _get_clinvar_path_filters(pathogenicity): - return { - f for f in (pathogenicity or {}).get(CLINVAR_KEY) or [] if f in CLINVAR_PATH_SIGNIFICANCES - } - - def _has_path_expr(self, ht, terms, field): - subfield, range_configs = self.PATHOGENICITY_FILTERS[field] - enum_lookup = self._get_enum_lookup(field, subfield) - - ranges = [[None, None]] - for path_filter, start, end in range_configs: - if path_filter in terms: - ranges[-1][1] = len(enum_lookup) if end is None else enum_lookup[end] - if ranges[-1][0] is None: - ranges[-1][0] = enum_lookup[start] - elif ranges[-1] != [None, None]: - ranges.append([None, None]) - - ranges = [r for r in ranges if r[0] is not None] - value = ht[field][f'{subfield}_id'] - return hl.any(lambda r: (value >= r[0]) & (value <= r[1]), ranges) - - def _format_results(self, ht, *args, **kwargs): - ht = ht.annotate(selected_transcript=self._selected_main_transcript_expr(ht)) - return super()._format_results(ht, *args, **kwargs) - - @classmethod - def _omim_sort(cls, r, omim_gene_set): - return [ - hl.if_else(omim_gene_set.contains(r.selected_transcript.gene_id), 0, 1), - ] + super()._omim_sort(r, omim_gene_set) - - @classmethod - def _gene_rank_sort(cls, r, gene_ranks): - return [gene_ranks.get(r.selected_transcript.gene_id)] + super()._gene_rank_sort(r, gene_ranks) - - def _import_variant_projects_ht(self, variant_id, *args, **kwargs): - # Get all the project-families for the looked up variant formatted as a dict of dicts: - # {: {: {: True}, : {: True}}, : ...} - lookup_ht = self._read_table('lookup.ht', skip_missing_field='project_stats') - lookup_ht = self._filter_variant_ids(lookup_ht, [variant_id]) - if lookup_ht is None: - raise HTTPNotFound() - variant_projects = lookup_ht.aggregate(hl.agg.take( - hl.dict(hl.enumerate(lookup_ht.project_stats).starmap(lambda i, ps: ( - lookup_ht.project_sample_types[i], - hl.enumerate(ps).starmap( - lambda j, s: hl.or_missing(self._stat_has_non_ref(s), j) - ).filter(hl.is_defined), - )).filter( - lambda x: x[1].any(hl.is_defined) - ).starmap(lambda project_key, family_indices: ( - project_key, - hl.dict(family_indices.map(lambda j: (lookup_ht.project_families[project_key][j], True))), - )).group_by( - lambda x: x[0][0] - ).map_values( - lambda project_data: hl.dict(project_data.starmap( - lambda project_key, families: (project_key[1], families) - )))), 1) - ) - - # Variant can be present in the lookup table with only ref calls, so is still not present in any projects - if not (variant_projects and variant_projects[0]): - raise HTTPNotFound() - variant_projects = variant_projects[0] - - self._has_both_sample_types = True - logger.info(f'Looking up {self.DATA_TYPE} variant in {len(variant_projects)} projects') - - projects_ht = super()._import_variant_projects_ht(variant_id, project_samples=variant_projects) - return projects_ht.select(familyGenotypes=hl.dict(projects_ht.family_entries.map( - lambda entries: (entries.first().familyGuid, self._get_sample_genotype(entries.filter(hl.is_defined))) - ))) - - @staticmethod - def _stat_has_non_ref(s): - return (s.heteroplasmic_samples > 0) | (s.homoplasmic_samples > 0) diff --git a/hail_search/queries/multi_data_types.py b/hail_search/queries/multi_data_types.py deleted file mode 100644 index 660bd70d0a..0000000000 --- a/hail_search/queries/multi_data_types.py +++ /dev/null @@ -1,216 +0,0 @@ -import hail as hl - -from hail_search.constants import ALT_ALT, REF_REF, CONSEQUENCE_SORT, OMIM_SORT, GROUPED_VARIANTS_FIELD, GENOME_VERSION_GRCh38 -from hail_search.queries.base import BaseHailTableQuery -from hail_search.queries.mito import MitoHailTableQuery -from hail_search.queries.snv_indel import SnvIndelHailTableQuery -from hail_search.queries.snv_indel_37 import SnvIndelHailTableQuery37 -from hail_search.queries.sv import SvHailTableQuery -from hail_search.queries.gcnv import GcnvHailTableQuery - -QUERY_CLASSES = [SnvIndelHailTableQuery, SnvIndelHailTableQuery37, MitoHailTableQuery, SvHailTableQuery, GcnvHailTableQuery] -QUERY_CLASS_MAP = {(cls.DATA_TYPE, cls.GENOME_VERSION): cls for cls in QUERY_CLASSES} -SNV_INDEL_DATA_TYPE = SnvIndelHailTableQuery.DATA_TYPE - - -class MultiDataTypeHailTableQuery(BaseHailTableQuery): - - LOADED_GLOBALS = True - - def __init__(self, sample_data, *args, **kwargs): - self._data_type_queries = {} - self._merged_comp_het_ht = None - self._sv_type_del_id = None - self._sv_data_type = next( - (dt for dt in [SvHailTableQuery.DATA_TYPE, GcnvHailTableQuery.DATA_TYPE] if dt in sample_data), None, - ) - super().__init__(sample_data, *args, **kwargs) - - def _load_filtered_table(self, sample_data, *args, **kwargs): - has_merged_comp_het = self._has_comp_het_search and SNV_INDEL_DATA_TYPE in sample_data and self._sv_data_type - overlapped_families = {s['family_guid'] for s in sample_data[SNV_INDEL_DATA_TYPE]}.intersection( - {s['family_guid'] for s in sample_data[self._sv_data_type]} - ) if has_merged_comp_het else None - - data_type_families = {} - # Multi project SV search skips projects with no matched variants, so run SV search first to reduce search space - for data_type in [self._sv_data_type, *sample_data.keys()]: - if data_type is None or data_type in self._data_type_queries: - continue - self._data_type_queries[data_type] = QUERY_CLASS_MAP[(data_type, GENOME_VERSION_GRCh38)]( - sample_data[data_type], *args, override_comp_het_alt=data_type == SNV_INDEL_DATA_TYPE, - overlapped_families=overlapped_families, sort=self._sort, sort_metadata=self._sort_metadata, - inheritance_mode=self._inheritance_mode, num_results=self._num_results, **kwargs - ) - if has_merged_comp_het and data_type in {self._sv_data_type, SNV_INDEL_DATA_TYPE}: - ht = self._data_type_queries[data_type].unfiltered_comp_het_ht - if ht is None: - has_merged_comp_het = False - overlapped_families = set() - else: - data_type_families[data_type] = hl.eval(ht.family_guids) - overlapped_families = overlapped_families.intersection(data_type_families[data_type]) - - if not (has_merged_comp_het and overlapped_families): - return - - variant_query = self._data_type_queries[SNV_INDEL_DATA_TYPE] - sv_query = self._data_type_queries[self._sv_data_type] - self.max_unaffected_samples = min(variant_query.max_unaffected_samples, sv_query.max_unaffected_samples) - merged_ht = self._filter_data_type_comp_hets( - variant_query, sv_query, sorted(overlapped_families), data_type_families, - ) - if merged_ht is not None: - self._merged_comp_het_ht = merged_ht.key_by() - - def _filter_data_type_comp_hets(self, variant_query, sv_query, overlapped_families, data_type_families): - variant_ht = self._family_filtered_ch_ht( - variant_query.unfiltered_comp_het_ht, overlapped_families, data_type_families[SNV_INDEL_DATA_TYPE], - ) - sv_ht = self._family_filtered_ch_ht( - sv_query.unfiltered_comp_het_ht, overlapped_families, data_type_families[self._sv_data_type], - ) - - sv_type_del_ids = sv_query.get_allowed_sv_type_ids([f'{getattr(sv_query, "SV_TYPE_PREFIX", "")}DEL']) - self._sv_type_del_id = list(sv_type_del_ids)[0] if sv_type_del_ids else None - - variant_samples_by_family = variant_query.entry_samples_by_family_guid - sv_samples_by_family = sv_query.entry_samples_by_family_guid - if any(f for f in overlapped_families if variant_samples_by_family[f] != sv_samples_by_family[f]): - sv_sample_indices = hl.array([[ - sv_samples_by_family[f].index(s) if s in sv_samples_by_family[f] else None - for s in variant_samples_by_family[f] - ] for f in overlapped_families]) - sv_ht = sv_ht.annotate(family_entries=hl.enumerate(sv_sample_indices).starmap( - lambda family_i, indices: hl.bind( - lambda family_entry: hl.or_missing( - hl.is_defined(family_entry), - indices.map(lambda sample_i: family_entry[sample_i]), - ), - sv_ht.family_entries[family_i], - ) - )) - - variant_ch_ht = variant_ht.group_by('gene_ids').aggregate(v1=hl.agg.collect(variant_ht.row)) - sv_ch_ht = sv_ht.group_by('gene_ids').aggregate(v2=hl.agg.collect(sv_ht.row)) - - variants = variant_ch_ht.join(sv_ch_ht).collect(_localize=False) - variants = variants.flatmap(lambda gvs: hl.rbind( - hl.set({gvs.gene_ids}), - lambda comp_het_gene_ids: gvs.v1.flatmap( - lambda v1: gvs.v2.map(lambda v2: hl.struct( - v1=v1.annotate(comp_het_gene_ids=comp_het_gene_ids), - v2=v2.annotate(comp_het_gene_ids=comp_het_gene_ids), - )) - ) - )) - variants = self._filter_comp_het_families(variants, set_secondary_annotations=False) - return hl.Table.parallelize(variants) - - @classmethod - def _family_filtered_ch_ht(cls, ht, overlapped_families, families): - if overlapped_families == families: - return ht - family_indices = hl.array([families.index(family_guid) for family_guid in overlapped_families]) - ht = ht.annotate(family_entries=family_indices.map(lambda i: ht.family_entries[i])) - return cls._apply_entry_filters(ht, select_globals=False) - - def _is_valid_comp_het_family(self, v1, v2, family_index): - is_valid = super()._is_valid_comp_het_family(v1, v2, family_index) - - # SNPs overlapped by trans deletions may be incorrectly called as hom alt, and should be - # considered comp hets with said deletions. Any other hom alt variants are not valid comp hets - entries_1 = v1.family_entries[family_index] - is_allowed_hom_alt = entries_1.all(lambda g: ~self.GENOTYPE_QUERY_MAP[ALT_ALT](g.GT)) | hl.all([ - v2.sv_type_id == self._sv_type_del_id, - v2.start_locus.position <= v1.locus.position, - v1.locus.position <= v2.end_locus.position, - ]) - return is_valid & is_allowed_hom_alt - - def _comp_het_entry_has_ref(self, gt1, gt2): - variant_query = self._data_type_queries[SNV_INDEL_DATA_TYPE] - sv_query = self._data_type_queries[self._sv_data_type] - return [variant_query.GENOTYPE_QUERY_MAP[REF_REF](gt1), sv_query.GENOTYPE_QUERY_MAP[REF_REF](gt2)] - - def format_search_ht(self): - hts = [] - for data_type, query in self._data_type_queries.items(): - dt_ht = query.format_search_ht() - if dt_ht is None: - continue - dt_ht = self._merged_sort(data_type, dt_ht) - hts.append(dt_ht.select('_sort', **{data_type: dt_ht.row})) - - if self._merged_comp_het_ht is not None: - ch_ht = self._merged_comp_het_ht.annotate( - v1=self._format_comp_het_result(self._merged_comp_het_ht.v1, SNV_INDEL_DATA_TYPE), - v2=self._format_comp_het_result(self._merged_comp_het_ht.v2, self._sv_data_type), - ) - hts.append(ch_ht.select( - _sort=hl.sorted([ch_ht.v1._sort.map(hl.float64), ch_ht.v2._sort.map(hl.float64)])[0], - **{f'comp_het_{self._sv_data_type}': ch_ht.row}, - )) - - if not hts: - return None - - ht = hts[0] - for sub_ht in hts[1:]: - ht = ht.union(sub_ht, unify=True) - - return ht - - def _format_comp_het_result(self, v, data_type): - result = self._data_type_queries[data_type]._format_results(v) - return self._merged_sort(data_type, result) - - def _merged_sort(self, data_type, ht): - # Certain sorts have an extra element for variant-type data, so need to add an element for SV data - if not data_type.startswith('SV'): - return ht - - sort_expr = None - if self._sort == CONSEQUENCE_SORT: - sort_expr = hl.array([hl.float64(4.5)]).extend(ht._sort.map(hl.float64)) - elif self._sort == OMIM_SORT: - sort_expr = hl.array([hl.int64(0)]).extend(ht._sort) - elif self._sort_metadata: - sort_expr = ht._sort[:1].extend(ht._sort) - - if sort_expr is not None: - ht = ht.annotate(_sort=sort_expr) - - return ht - - def _format_collected_rows(self, collected): - data_types = [*self._data_type_queries] - if self._merged_comp_het_ht: - data_types.append(f'comp_het_{self._sv_data_type}') - return super()._format_collected_rows([self._format_collected_row(row, data_types) for row in collected]) - - @staticmethod - def _format_collected_row( row, data_types): - data_type = next(data_type for data_type in data_types if row.get(data_type)) - formatted_row = row.get(data_type) - if 'comp_het' in data_type: - formatted_row = {GROUPED_VARIANTS_FIELD: sorted([formatted_row.v1, formatted_row.v2], key=lambda x: x._sort)} - return formatted_row - - def format_gene_count_hts(self): - hts = [] - for query in self._data_type_queries.values(): - hts += query.format_gene_count_hts() - if self._merged_comp_het_ht is not None: - hts += [ - self._comp_het_gene_count_ht(self._merged_comp_het_ht, 'v1', SNV_INDEL_DATA_TYPE), - self._comp_het_gene_count_ht(self._merged_comp_het_ht, 'v2', self._sv_data_type) - ] - return hts - - def _comp_het_gene_count_ht(self, ht, field, data_type): - selects = { - **self._gene_count_selects(), - 'gene_ids': self._data_type_queries[data_type]._gene_ids_expr, - } - return ht.select(**{k: v(ht[field]) for k, v in selects.items()}) diff --git a/hail_search/queries/snv_indel.py b/hail_search/queries/snv_indel.py deleted file mode 100644 index f83bac64db..0000000000 --- a/hail_search/queries/snv_indel.py +++ /dev/null @@ -1,104 +0,0 @@ -from collections import OrderedDict -import hail as hl - -from hail_search.constants import GENOME_VERSION_GRCh38, SCREEN_KEY, PREFILTER_FREQ_CUTOFF, ALPHAMISSENSE_SORT, \ - UTR_ANNOTATOR_KEY, EXTENDED_SPLICE_KEY, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY, GENOME_VERSION_GRCh37 -from hail_search.queries.base import BaseHailTableQuery, PredictionPath -from hail_search.queries.snv_indel_37 import SnvIndelHailTableQuery37 - -EXTENDED_SPLICE_REGION_CONSEQUENCE = 'extended_intronic_splice_region_variant' - - -class SnvIndelHailTableQuery(SnvIndelHailTableQuery37): - - GENOME_VERSION = GENOME_VERSION_GRCh38 - LIFT_GENOME_VERSION = GENOME_VERSION_GRCh37 - PREDICTION_FIELDS_CONFIG = { - **SnvIndelHailTableQuery37.PREDICTION_FIELDS_CONFIG, - 'fathmm': PredictionPath('dbnsfp', 'fathmm_MKL_coding_score'), - 'mut_pred': PredictionPath('dbnsfp', 'MutPred_score'), - 'vest': PredictionPath('dbnsfp', 'VEST4_score'), - 'gnomad_noncoding': PredictionPath('gnomad_non_coding_constraint', 'z_score'), - } - LIFTOVER_ANNOTATION_FIELDS = BaseHailTableQuery.LIFTOVER_ANNOTATION_FIELDS - ANNOTATION_OVERRIDE_FIELDS = SnvIndelHailTableQuery37.ANNOTATION_OVERRIDE_FIELDS + [ - SCREEN_KEY, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY, - ] - SECONDARY_ANNOTATION_OVERRIDE_FIELDS = [MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY] - FREQUENCY_PREFILTER_FIELDS = OrderedDict([ - (True, 0.001), - ('is_gt_1_percent', PREFILTER_FREQ_CUTOFF), - ('is_gt_3_percent', 0.03), - ('is_gt_5_percent', 0.05), - ('is_gt_10_percent', 0.1), - ]) - SORTS = { - **SnvIndelHailTableQuery37.SORTS, - ALPHAMISSENSE_SORT: lambda r: [ - SnvIndelHailTableQuery37._format_prediction_sort_value( - hl.min(r.sorted_transcript_consequences.map(lambda t: t.alphamissense.pathogenicity)) - ), - SnvIndelHailTableQuery37._format_prediction_sort_value(r.selected_transcript.alphamissense.pathogenicity), - ], - } - - def _get_allowed_consequence_ids(self, annotations): - parsed_allowed_consequences = {} - allowed_consequence_ids = super()._get_allowed_consequence_ids(annotations) - if allowed_consequence_ids: - parsed_allowed_consequences[self.TRANSCRIPT_CONSEQUENCE_FIELD] = allowed_consequence_ids - - utr_consequence_ids = self._get_enum_terms_ids( - self.TRANSCRIPTS_FIELD, subfield='utrannotator', nested_subfield='fiveutr_consequence', - terms=(annotations.get(UTR_ANNOTATOR_KEY) or []), - ) - if utr_consequence_ids: - parsed_allowed_consequences[UTR_ANNOTATOR_KEY] = utr_consequence_ids - - if EXTENDED_SPLICE_REGION_CONSEQUENCE in (annotations.get(EXTENDED_SPLICE_KEY) or []): - parsed_allowed_consequences[EXTENDED_SPLICE_REGION_CONSEQUENCE] = True - - return parsed_allowed_consequences - - @staticmethod - def _get_allowed_transcripts_filter(allowed_consequence_ids): - allowed_consequence_filters = [] - - consequence_ids = allowed_consequence_ids.get(SnvIndelHailTableQuery37.TRANSCRIPT_CONSEQUENCE_FIELD) - if consequence_ids: - allowed_consequence_filters.append(SnvIndelHailTableQuery37._get_allowed_transcripts_filter(consequence_ids)) - - utr_consequences = allowed_consequence_ids.get(UTR_ANNOTATOR_KEY) - if utr_consequences: - utr_consequences = hl.set(utr_consequences) - allowed_consequence_filters.append(lambda tc: utr_consequences.contains(tc.utrannotator.fiveutr_consequence_id)) - - if allowed_consequence_ids.get(EXTENDED_SPLICE_REGION_CONSEQUENCE): - allowed_consequence_filters.append(lambda tc: tc.spliceregion.extended_intronic_splice_region_variant) - - return allowed_consequence_filters[0] if len(allowed_consequence_filters) == 1 else lambda tc: hl.any([ - f(tc) for f in allowed_consequence_filters - ]) - - def _get_annotation_override_filters(self, ht, annotation_overrides): - annotation_filters = super()._get_annotation_override_filters(ht, annotation_overrides) - - if annotation_overrides.get(SCREEN_KEY): - allowed_consequences = hl.set(self._get_enum_terms_ids(SCREEN_KEY.lower(), 'region_type', annotation_overrides[SCREEN_KEY])) - annotation_filters.append(allowed_consequences.contains(ht.screen.region_type_ids.first())) - - for feature_key in [MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY]: - if annotation_overrides.get(feature_key): - field = f'sorted_{feature_key}_consequences' - allowed_consequences = hl.set(self._get_enum_terms_ids( - field, self.TRANSCRIPT_CONSEQUENCE_FIELD, annotation_overrides[feature_key]), - ) - annotation_filters.append( - ht[field].any(lambda c: c.consequence_term_ids.any(allowed_consequences.contains)) - ) - - return annotation_filters - - @staticmethod - def _lookup_variant_annotations(): - return {'liftover_locus': lambda r: r.rg37_locus} diff --git a/hail_search/queries/snv_indel_37.py b/hail_search/queries/snv_indel_37.py deleted file mode 100644 index 31ec40f3fc..0000000000 --- a/hail_search/queries/snv_indel_37.py +++ /dev/null @@ -1,160 +0,0 @@ -from aiohttp.web import HTTPNotFound -from collections import OrderedDict -import hail as hl - -from hail_search.constants import CLINVAR_KEY, HGMD_KEY, HGMD_PATH_RANGES, \ - GNOMAD_GENOMES_FIELD, PREFILTER_FREQ_CUTOFF, PATH_FREQ_OVERRIDE_CUTOFF, PATHOGENICTY_HGMD_SORT_KEY, \ - SPLICE_AI_FIELD, GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38 -from hail_search.queries.base import PredictionPath, QualityFilterFormat -from hail_search.queries.mito import MitoHailTableQuery - - -class SnvIndelHailTableQuery37(MitoHailTableQuery): - - DATA_TYPE = 'SNV_INDEL' - GENOME_VERSION = GENOME_VERSION_GRCh37 - LIFT_GENOME_VERSION = GENOME_VERSION_GRCh38 - - GENOTYPE_FIELDS = {f.lower(): f for f in ['DP', 'GQ', 'AB']} - QUALITY_FILTER_FORMAT = { - 'AB': QualityFilterFormat(override=lambda gt: ~gt.GT.is_het(), scale=100), - } - POPULATIONS = { - 'seqr': {'af': None, 'an': None, 'hom': 'hom', 'hemi': None, 'het': None, 'sort': 'callset_af', 'sort_subfield': 'ac'}, - 'topmed': {'hemi': None}, - 'exac': { - 'filter_af': 'AF_POPMAX', 'ac': 'AC_Adj', 'an': 'AN_Adj', 'hom': 'AC_Hom', 'hemi': 'AC_Hemi', - 'het': 'AC_Het', - }, - 'gnomad_exomes': {'filter_af': 'AF_POPMAX_OR_GLOBAL', 'het': None, 'sort': 'gnomad_exomes'}, - GNOMAD_GENOMES_FIELD: {'filter_af': 'AF_POPMAX_OR_GLOBAL', 'het': None, 'sort': 'gnomad'}, - } - PREDICTION_FIELDS_CONFIG = { - 'cadd': PredictionPath('dbnsfp', 'CADD_phred'), - 'eigen': PredictionPath('eigen', 'Eigen_phred'), - 'mpc': PredictionPath('dbnsfp', 'MPC_score'), - 'primate_ai': PredictionPath('dbnsfp', 'PrimateAI_score'), - SPLICE_AI_FIELD: PredictionPath(SPLICE_AI_FIELD, 'delta_score'), - 'splice_ai_consequence': PredictionPath(SPLICE_AI_FIELD, 'splice_consequence'), - 'mut_taster': PredictionPath('dbnsfp', 'MutationTaster_pred'), - 'polyphen': PredictionPath('dbnsfp', 'Polyphen2_HVAR_score'), - 'revel': PredictionPath('dbnsfp', 'REVEL_score'), - 'sift': PredictionPath('dbnsfp', 'SIFT_score'), - } - PATHOGENICITY_FILTERS = { - **MitoHailTableQuery.PATHOGENICITY_FILTERS, - HGMD_KEY: ('class', HGMD_PATH_RANGES), - } - ANNOTATION_OVERRIDE_FIELDS = [SPLICE_AI_FIELD] - - CORE_FIELDS = MitoHailTableQuery.CORE_FIELDS + ['CAID'] - - LIFTOVER_ANNOTATION_FIELDS = {} - BASE_ANNOTATION_FIELDS = { - k: v for k, v in MitoHailTableQuery.BASE_ANNOTATION_FIELDS.items() - if k not in MitoHailTableQuery.MITO_ANNOTATION_FIELDS - } - ENUM_ANNOTATION_FIELDS = { - **MitoHailTableQuery.ENUM_ANNOTATION_FIELDS, - 'screen': { - 'response_key': 'screenRegionType', - 'format_value': lambda value: value.region_types.first(), - }, - } - - SORTS = { - **MitoHailTableQuery.SORTS, - PATHOGENICTY_HGMD_SORT_KEY: lambda r: [MitoHailTableQuery.CLINVAR_SORT(CLINVAR_KEY, r), r.hgmd.class_id], - } - - FREQUENCY_PREFILTER_FIELDS = OrderedDict([ - (True, PREFILTER_FREQ_CUTOFF), - ('is_gt_10_percent', 0.1), - ]) - - PREFILTER_TABLES = { - **MitoHailTableQuery.PREFILTER_TABLES, - GNOMAD_GENOMES_FIELD: 'high_af_variants.ht', - } - - def _prefilter_entries_table(self, ht, *args, raw_intervals=None, **kwargs): - ht = super()._prefilter_entries_table(ht, *args, **kwargs) - load_table_intervals = self._load_table_kwargs.get('_intervals') or [] - no_interval_prefilter = not load_table_intervals or len(raw_intervals or []) > len(load_table_intervals) - if 'variant_ht' not in self._load_table_kwargs and no_interval_prefilter: - af_ht = self._get_loaded_filter_ht( - GNOMAD_GENOMES_FIELD, self._get_gnomad_af_prefilter, **kwargs) - if af_ht: - ht = ht.filter(hl.is_missing(af_ht[ht.key])) - return ht - - def _get_gnomad_af_prefilter(self, frequencies=None, pathogenicity=None, **kwargs): - gnomad_genomes_filter = (frequencies or {}).get(GNOMAD_GENOMES_FIELD, {}) - af_cutoff = gnomad_genomes_filter.get('af') - if af_cutoff is None and gnomad_genomes_filter.get('ac') is not None: - af_cutoff = PREFILTER_FREQ_CUTOFF - if af_cutoff is None: - return False - - af_cutoff_field = self._get_af_prefilter_field(af_cutoff) - if af_cutoff_field is None: - return False - - clinvar_path_ht = False - if af_cutoff < PATH_FREQ_OVERRIDE_CUTOFF: - clinvar_path_ht = self._get_loaded_clinvar_prefilter_ht(pathogenicity) - - if clinvar_path_ht is not False: - path_cutoff_field = self._get_af_prefilter_field(PATH_FREQ_OVERRIDE_CUTOFF) - non_clinvar_filter = lambda ht: hl.is_missing(clinvar_path_ht[ht.key]) - if af_cutoff_field is not True: - non_clinvar_var_filter = non_clinvar_filter - non_clinvar_filter = lambda ht: non_clinvar_var_filter(ht) & self._af_prefilter(af_cutoff_field)(ht) - af_filter = lambda ht: ht[path_cutoff_field] | non_clinvar_filter(ht) - else: - af_filter = self._af_prefilter(af_cutoff_field) - - return af_filter - - @staticmethod - def _af_prefilter(af_cutoff_field): - return True if af_cutoff_field is True else lambda ht: ht[af_cutoff_field] - - def _get_af_prefilter_field(self, af_cutoff): - return next((field for field, cutoff in self.FREQUENCY_PREFILTER_FIELDS.items() if af_cutoff <= cutoff), None) - - def _get_annotation_override_filters(self, ht, annotation_overrides): - annotation_filters = super()._get_annotation_override_filters(ht, annotation_overrides) - - if annotation_overrides.get(SPLICE_AI_FIELD): - score_filter, _ = self._get_in_silico_filter(ht, SPLICE_AI_FIELD, annotation_overrides[SPLICE_AI_FIELD]) - annotation_filters.append(score_filter) - - return annotation_filters - - @staticmethod - def _stat_has_non_ref(s): - return (s.het_samples > 0) | (s.hom_samples > 0) - - @staticmethod - def _lookup_variant_annotations(): - return {'liftover_locus': lambda r: r.rg38_locus} - - @classmethod - def _get_lifted_table_path(cls, path): - return f'{cls._get_table_dir(path)}/{cls.LIFT_GENOME_VERSION}/{cls.DATA_TYPE}/{path}' - - def _get_variant_project_data(self, variant_id, variant=None, **kwargs): - project_data = super()._get_variant_project_data(variant_id, **kwargs) - liftover_locus = variant.pop('liftover_locus') - if not liftover_locus: - return project_data - interval = hl.eval(hl.interval(liftover_locus, liftover_locus, includes_start=True, includes_end=True)) - self._load_table_kwargs['_intervals'] = [interval] - self._get_table_path = self._get_lifted_table_path - try: - lift_project_data = super()._get_variant_project_data(variant_id, **kwargs) - except HTTPNotFound: - return project_data - project_data['familyGenotypes'].update(lift_project_data['familyGenotypes']) - return project_data.annotate(liftedFamilyGuids=sorted(lift_project_data['familyGenotypes'].keys())) diff --git a/hail_search/queries/sv.py b/hail_search/queries/sv.py deleted file mode 100644 index 01df7afc57..0000000000 --- a/hail_search/queries/sv.py +++ /dev/null @@ -1,172 +0,0 @@ -import hail as hl - - -from hail_search.constants import CONSEQUENCE_SORT, NEW_SV_FIELD, STRUCTURAL_ANNOTATION_FIELD, FAMILY_GUID_FIELD, \ - GENOTYPES_FIELD -from hail_search.queries.base import BaseHailTableQuery, PredictionPath - - -class SvHailTableQuery(BaseHailTableQuery): - - DATA_TYPE = 'SV_WGS' - KEY_FIELD = ('variant_id',) - - GENOTYPE_FIELDS = {f.lower(): f for f in ['CN', 'GQ']} - COMPUTED_GENOTYPE_FIELDS = { - k: lambda entry, field, *args: entry.concordance[field] for k in ['new_call', 'prev_call', 'prev_num_alt'] - } - GENOTYPE_QUERY_FIELDS = {'gq_sv': 'GQ', 'gq': None} - - TRANSCRIPTS_FIELD = 'sorted_gene_consequences' - TRANSCRIPT_CONSEQUENCE_FIELD = 'major_consequence' - CORE_FIELDS = BaseHailTableQuery.CORE_FIELDS + ['algorithms'] - BASE_ANNOTATION_FIELDS = { - 'bothsidesSupport': lambda r: r.bothsides_support, - 'chrom': lambda r: r.start_locus.contig.replace('^chr', ''), - 'pos': lambda r: r.start_locus.position, - 'end': lambda r: r.end_locus.position, - 'rg37LocusEnd': lambda r: hl.or_missing( - hl.is_defined(r.rg37_locus_end), hl.struct(contig=r.rg37_locus_end.contig, position=r.rg37_locus_end.position) - ), - **BaseHailTableQuery.BASE_ANNOTATION_FIELDS, - } - ENUM_ANNOTATION_FIELDS = { - TRANSCRIPTS_FIELD: BaseHailTableQuery.ENUM_ANNOTATION_FIELDS['transcripts'], - } - - POPULATIONS = { - 'sv_callset': {'hemi': None, 'sort': 'callset_af', 'sort_subfield': 'ac'}, - 'gnomad_svs': {'id': 'ID', 'ac': None, 'an': None, 'hom': 'N_HOM', 'hemi': None, 'het': 'N_HET', 'sort': 'gnomad'}, - } - POPULATION_FIELDS = {'sv_callset': 'gt_stats'} - PREDICTION_FIELDS_CONFIG = { - 'strvctvre': PredictionPath('strvctvre', 'score'), - } - ANNOTATION_OVERRIDE_FIELDS = [STRUCTURAL_ANNOTATION_FIELD] - SECONDARY_ANNOTATION_OVERRIDE_FIELDS = [STRUCTURAL_ANNOTATION_FIELD] - - SORTS = { - **BaseHailTableQuery.SORTS, - CONSEQUENCE_SORT: lambda r: [hl.min(r.sorted_gene_consequences.map(lambda g: g.major_consequence_id))], - 'size': lambda r: [hl.if_else( - r.start_locus.contig == r.end_locus.contig, r.start_locus.position - r.end_locus.position, -50, - )], - } - - def __init__(self, *args, padded_interval=None, **kwargs): - self._is_interval_filtered = bool(padded_interval) - super().__init__(*args, padded_interval=padded_interval, **kwargs) - - def _set_interval_prefilter(self, *args, **kwargs): - self._is_interval_filtered = True - - @classmethod - def _get_sample_type(cls, *args): - return cls.DATA_TYPE.split('_')[-1] - - def _read_project_table(self, project_guid: str, sample_type: str, **kwargs): - ht = super()._read_project_table(project_guid, sample_type, **kwargs) - if ht is not None: - ht = ht.annotate_globals(sample_type=sample_type) - return ht - - def import_filtered_table(self, project_samples, *args, **kwargs): - if len(project_samples) == 1 or not self._is_interval_filtered: - return super().import_filtered_table(project_samples, *args, **kwargs) - - ht = self._read_table('annotations.ht') - ht = self._filter_annotated_table(ht, is_comp_het=self._has_comp_het_search, **kwargs) - - self._load_table_kwargs['variant_ht'] = ht.select() - families_ht, comp_het_families_ht = self._import_families_tables(project_samples, *args, skip_missing_field='family_entries', **kwargs) - - if comp_het_families_ht is not None: - self._comp_het_ht = self._annotate_families_table_annotations(comp_het_families_ht, ht, is_comp_het=True) - self._comp_het_ht = self._filter_compound_hets() - - if families_ht is not None: - self._ht = self._annotate_families_table_annotations(families_ht, ht) - if self._has_secondary_annotations: - self._ht = self._ht.filter(hl.any(self._get_annotation_filters(self._ht))) - - def _annotate_families_table_annotations(self, families_ht, annotations_ht, is_comp_het=False): - return families_ht.annotate(**annotations_ht[families_ht.key]) - - def _filter_annotated_table(self, ht, *args, parsed_intervals=None, exclude_intervals=False, padded_interval=None, gene_ids=None, **kwargs): - if parsed_intervals and len(parsed_intervals) != len(gene_ids or []): - interval_filter = hl.array(parsed_intervals).any(lambda interval: hl.if_else( - ht.start_locus.contig == ht.end_locus.contig, - interval.overlaps(hl.interval(ht.start_locus, ht.end_locus)), - interval.contains(ht.start_locus) | interval.contains(ht.end_locus), - )) - if exclude_intervals: - interval_filter = ~interval_filter - ht = ht.filter(interval_filter) - if padded_interval: - padding = int((padded_interval['end'] - padded_interval['start']) * padded_interval['padding']) - chrom = f"chr{padded_interval['chrom']}" - ht = ht.filter(hl.all([ - self._locus_in_range(ht.start_locus, chrom, padded_interval['start'], padding), - self._locus_in_range(ht.end_locus, chrom, padded_interval['end'], padding) - ])) - - return super()._filter_annotated_table(ht, *args, gene_ids=gene_ids, **kwargs) - - def _locus_in_range(self, locus, chrom, position, padding): - window = hl.locus(chrom, position, reference_genome=self.GENOME_VERSION).window(padding, padding) - return window.contains(locus) - - def _parse_annotations(self, annotations, *args, **kwargs): - parsed_annotations = super()._parse_annotations(annotations, *args, **kwargs) - parsed_annotations[NEW_SV_FIELD] = (annotations or {}).get(NEW_SV_FIELD) - return parsed_annotations - - def _get_family_passes_quality_filter(self, quality_filter, ht, parsed_annotations=None, **kwargs): - passes_quality = super()._get_family_passes_quality_filter(quality_filter, ht) - if not (parsed_annotations or {}).get(NEW_SV_FIELD): - return passes_quality - - entries_has_new_call = lambda entries: entries.any(lambda x: x.concordance.new_call) - if passes_quality is None: - return entries_has_new_call - - return lambda entries: entries_has_new_call(entries) & passes_quality(entries) - - def _get_annotation_override_filters(self, ht, annotation_overrides): - annotation_filters = [] - if annotation_overrides.get(STRUCTURAL_ANNOTATION_FIELD): - allowed_type_ids = self.get_allowed_sv_type_ids(annotation_overrides[STRUCTURAL_ANNOTATION_FIELD]) - if allowed_type_ids: - annotation_filters.append(hl.set(allowed_type_ids).contains(ht.sv_type_id)) - - return annotation_filters - - def get_allowed_sv_type_ids(self, sv_types): - return self._get_enum_terms_ids('sv_type', None, sv_types) - - def _additional_annotation_fields(self): - sv_type_enum = self._enums['sv_type'] - insertion_type_id = sv_type_enum.index('INS') - get_end_chrom = lambda r: hl.or_missing(r.start_locus.contig != r.end_locus.contig, r.end_locus.contig.replace('^chr', '')) - return { - 'cpxIntervals': lambda r: self._format_enum( - r, 'cpx_intervals', {'type': sv_type_enum}, annotate_value=lambda val, *args: { - 'chrom': val.start.contig, - 'start': val.start.position, - 'end': val.end.position, - }, - ), - # For insertions, end_locus represents the svSourceDetail, otherwise represents the endChrom - 'endChrom': lambda r: hl.or_missing(r.sv_type_id != insertion_type_id, get_end_chrom(r)), - 'svSourceDetail': lambda r: hl.or_missing(r.sv_type_id == insertion_type_id, hl.bind( - lambda end_chrom: hl.or_missing(hl.is_defined(end_chrom), hl.struct(chrom=end_chrom)), - get_end_chrom(r), - )), - } - - def _import_variant_projects_ht(self, variant_id, sample_data=None, **kwargs): - project_samples, _ = self._parse_sample_data(sample_data) - projects_ht = super()._import_variant_projects_ht(variant_id, project_samples=project_samples) - - annotation_fields = self.annotation_fields(include_genotype_overrides=False) - return projects_ht.select(**{k: annotation_fields[k](projects_ht) for k in [FAMILY_GUID_FIELD, GENOTYPES_FIELD]}) diff --git a/hail_search/requirements-test.in b/hail_search/requirements-test.in deleted file mode 100644 index 245d0d3637..0000000000 --- a/hail_search/requirements-test.in +++ /dev/null @@ -1,2 +0,0 @@ -coverage<5.2 -pytest-aiohttp diff --git a/hail_search/requirements-test.txt b/hail_search/requirements-test.txt deleted file mode 100644 index fa1e1672a2..0000000000 --- a/hail_search/requirements-test.txt +++ /dev/null @@ -1,50 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# To update, run: -# -# pip-compile hail_search/requirements-test.in -# -aiohappyeyeballs==2.3.5 - # via aiohttp -aiohttp==3.10.11 - # via pytest-aiohttp -aiosignal==1.3.1 - # via aiohttp -async-timeout==4.0.2 - # via aiohttp -attrs==23.1.0 - # via aiohttp -coverage==5.1 - # via -r requirements-test.in -exceptiongroup==1.1.3 - # via pytest -frozenlist==1.3.3 - # via - # aiohttp - # aiosignal -idna==3.7 - # via yarl -iniconfig==2.0.0 - # via pytest -multidict==6.0.4 - # via - # aiohttp - # yarl -packaging==23.1 - # via pytest -pluggy==1.2.0 - # via pytest -propcache==0.2.0 - # via yarl -pytest==7.4.0 - # via - # pytest-aiohttp - # pytest-asyncio -pytest-aiohttp==1.0.4 - # via -r requirements-test.in -pytest-asyncio==0.21.0 - # via pytest-aiohttp -tomli==2.0.1 - # via pytest -yarl==1.17.2 - # via aiohttp diff --git a/hail_search/search.py b/hail_search/search.py deleted file mode 100644 index 8f43137191..0000000000 --- a/hail_search/search.py +++ /dev/null @@ -1,39 +0,0 @@ -from hail_search.queries.multi_data_types import QUERY_CLASS_MAP, SNV_INDEL_DATA_TYPE, MultiDataTypeHailTableQuery - - -def search_hail_backend(request, gene_counts=False): - sample_data = request.pop('sample_data', {}) - genome_version = request.pop('genome_version') - - data_types = list(sample_data.keys()) - single_data_type = data_types[0] if len(data_types) == 1 else None - - if single_data_type: - sample_data = sample_data[single_data_type] - query_cls = QUERY_CLASS_MAP[(single_data_type, genome_version)] - else: - query_cls = MultiDataTypeHailTableQuery - - query = query_cls(sample_data, **request) - if gene_counts: - return query.gene_counts() - else: - return query.search() - - -def lookup_variant(request): - data_type = request.get('data_type', SNV_INDEL_DATA_TYPE) - query = QUERY_CLASS_MAP[(data_type, request['genome_version'])](sample_data=None) - return query.lookup_variant(request['variant_id'], sample_data=request.get('sample_data')) - - -def lookup_variants(request): - query = QUERY_CLASS_MAP[(request['data_type'], request['genome_version'])](sample_data=None) - return query.lookup_variants(request['variant_ids']) - - -def load_globals(): - return { - str(k): v.load_globals() - for k, v in QUERY_CLASS_MAP.items() - } diff --git a/hail_search/test_search.py b/hail_search/test_search.py deleted file mode 100644 index cd3235e79c..0000000000 --- a/hail_search/test_search.py +++ /dev/null @@ -1,1293 +0,0 @@ -from aiohttp.test_utils import AioHTTPTestCase -import asyncio -from copy import deepcopy -import hail as hl -import time -from unittest import mock - -from hail_search.test_utils import get_hail_search_body, FAMILY_2_VARIANT_SAMPLE_DATA, FAMILY_2_MISSING_SAMPLE_DATA, \ - VARIANT1, VARIANT2, VARIANT3, VARIANT4, MULTI_PROJECT_SAMPLE_DATA, MULTI_PROJECT_MISSING_SAMPLE_DATA, \ - LOCATION_SEARCH, EXCLUDE_LOCATION_SEARCH, VARIANT_ID_SEARCH, RSID_SEARCH, GENE_COUNTS, SV_WGS_SAMPLE_DATA, \ - SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, \ - GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, SV_WES_SAMPLE_DATA, EXPECTED_SAMPLE_DATA, \ - FAMILY_2_MITO_SAMPLE_DATA, FAMILY_2_ALL_SAMPLE_DATA, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, \ - EXPECTED_SAMPLE_DATA_WITH_SEX, SV_WGS_SAMPLE_DATA_WITH_SEX, VARIANT_LOOKUP_VARIANT, \ - MULTI_PROJECT_SAMPLE_TYPES_SAMPLE_DATA, FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA, \ - VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA_MISSING_PARENTAL_WGS, \ - VARIANT4_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, \ - VARIANT1_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, \ - VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, VARIANT3_BOTH_SAMPLE_TYPES, PROJECT_2_VARIANT, GRCH37_VARIANT -from hail_search.web_app import init_web_app, sync_to_async_hail_query -from hail_search.queries.base import BaseHailTableQuery - -PROJECT_2_VARIANT_BOTH_SAMPLE_TYPES = deepcopy(PROJECT_2_VARIANT) -PROJECT_2_VARIANT_BOTH_SAMPLE_TYPES['genotypes']['I000015_na20885'] = { - 'sampleId': 'NA20885', 'sampleType': 'WGS', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'dp': 8, 'gq': 14, 'ab': 0.875, 'filters': [], -} - -FAMILY_3_VARIANT = deepcopy(VARIANT3) -FAMILY_3_VARIANT['familyGuids'] = ['F000003_3'] -FAMILY_3_VARIANT['genotypes'] = { - 'I000007_na20870': { - 'sampleId': 'NA20870', 'sampleType': 'WES', 'individualGuid': 'I000007_na20870', 'familyGuid': 'F000003_3', - 'numAlt': 1, 'dp': 28, 'gq': 99, 'ab': 0.6785714285714286, 'filters': [], - }, -} - -MULTI_FAMILY_VARIANT = deepcopy(VARIANT3) -MULTI_FAMILY_VARIANT['familyGuids'] += FAMILY_3_VARIANT['familyGuids'] -MULTI_FAMILY_VARIANT['genotypes'].update(FAMILY_3_VARIANT['genotypes']) - -SELECTED_TRANSCRIPT_MULTI_FAMILY_VARIANT = {**MULTI_FAMILY_VARIANT, 'selectedMainTranscriptId': 'ENST00000426137'} -SELECTED_ANNOTATION_TRANSCRIPT_MULTI_FAMILY_VARIANT = {**MULTI_FAMILY_VARIANT, 'selectedMainTranscriptId': 'ENST00000497611'} -SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4 = {**VARIANT4, 'selectedMainTranscriptId': 'ENST00000350997'} -SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_3 = {**VARIANT3, 'selectedMainTranscriptId': 'ENST00000497611'} -SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2 = {**VARIANT2, 'selectedMainTranscriptId': 'ENST00000459627'} -MULTI_DATA_TYPE_COMP_HET_VARIANT2 = {**VARIANT2, 'selectedMainTranscriptId': 'ENST00000450625'} - -PROJECT_2_VARIANT1 = deepcopy(VARIANT1) -PROJECT_2_VARIANT1['familyGuids'] = ['F000011_11'] -PROJECT_2_VARIANT1['genotypes'] = { - 'I000015_na20885': { - 'sampleId': 'NA20885', 'sampleType': 'WES', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 2, 'dp': 6, 'gq': 16, 'ab': 1.0, 'filters': [], - }, -} -MULTI_PROJECT_VARIANT1 = deepcopy(VARIANT1) -MULTI_PROJECT_VARIANT1['familyGuids'] += PROJECT_2_VARIANT1['familyGuids'] -MULTI_PROJECT_VARIANT1['genotypes'].update(PROJECT_2_VARIANT1['genotypes']) -MULTI_PROJECT_VARIANT2 = deepcopy(VARIANT2) -MULTI_PROJECT_VARIANT2['familyGuids'].append('F000011_11') -MULTI_PROJECT_VARIANT2['genotypes']['I000015_na20885'] = { - 'sampleId': 'NA20885', 'sampleType': 'WES', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'dp': 28, 'gq': 99, 'ab': 0.5, 'filters': [], -} - -NO_GENOTYPE_GCNV_VARIANT = {**GCNV_VARIANT4, 'numExon': 8, 'end': 38736268} - -# Ensures no variants are filtered out by annotation/path filters for compound hets -COMP_HET_ALL_PASS_FILTERS = { - 'annotations': {'splice_ai': '0.0', 'structural': ['DEL', 'CPX', 'INS', 'gCNV_DEL', 'gCNV_DUP']}, - 'pathogenicity': {'clinvar': ['likely_pathogenic']}, -} - -NEW_SV_FILTER = {'new_structural_variants': ['NEW']} - -SV_GENE_COUNTS = { - 'ENSG00000171621': {'total': 2, 'families': {'F000011_11': 2}}, - 'ENSG00000083544': {'total': 1, 'families': {'F000011_11': 1}}, - 'ENSG00000184986': {'total': 1, 'families': {'F000011_11': 1}}, - 'null': {'total': 1, 'families': {'F000011_11': 1}}, -} - -GCNV_GENE_COUNTS = { - 'ENSG00000103495': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000167371': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000280893': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000275023': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000277972': {'total': 1, 'families': {'F000002_2': 1}}, -} - -OMIM_SORT_METADATA = ['ENSG00000177000', 'ENSG00000097046', 'ENSG00000275023'] - -MULTI_PROJECT_BOTH_SAMPLE_TYPE_VARIANTS = [ - deepcopy(v) for v in - [PROJECT_2_VARIANT, MULTI_PROJECT_VARIANT1, MULTI_PROJECT_VARIANT2, VARIANT3, VARIANT4] -] -for v in MULTI_PROJECT_BOTH_SAMPLE_TYPE_VARIANTS: - for indiv_id, gts in v['genotypes'].items(): - v['genotypes'][indiv_id] = [gts] -for v in MULTI_PROJECT_BOTH_SAMPLE_TYPE_VARIANTS[:-2]: - v['genotypes']['I000015_na20885'].append({**v['genotypes']['I000015_na20885'][0], 'sampleType': 'WGS'}) - -SECOND_PROJECT_SV_WES_SAMPLE_DATA = [{**s, 'sample_type': 'WES'} for s in SV_WGS_SAMPLE_DATA['SV_WGS']] - -MULTI_PROJECT_GCNV_VARIANT3 = { - **GCNV_VARIANT3, - 'familyGuids': GCNV_VARIANT3['familyGuids'] + ['F000011_11'], - 'genotypes': { - **GCNV_VARIANT3['genotypes'], - 'I000015_na20885': { - 'sampleId': 'NA20885', 'sampleType': 'WES', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 2, 'cn': 4, 'qs': 27, 'defragged': True, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': True, 'prevCall': False, 'prevOverlap': False, 'filters': [], - }, - 'I000035_na20883': { - 'sampleId': 'NA20883', 'sampleType': 'WES', 'individualGuid': 'I000035_na20883', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'cn': 3, 'qs': 51, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': False, 'prevOverlap': True, 'filters': [], - }, - 'I000025_na20884': { - 'sampleId': 'NA20884', 'sampleType': 'WES', 'individualGuid': 'I000025_na20884', 'familyGuid': 'F000011_11', 'numAlt': 0, - 'cn': None, 'qs': None, 'defragged': None, 'start': None, 'end': None, 'numExon': None, 'geneIds': None, - 'newCall': None, 'prevCall': None, 'prevOverlap': None, 'filters': [], - }, - }, -} - -def _sorted(variant, sorts): - return {**variant, '_sort': sorts + variant['_sort']} - - -class HailSearchTestCase(AioHTTPTestCase): - - async def get_application(self): - return await init_web_app() - - async def test_sync_to_async_hail_query(self): - request = mock.Mock() - request.app = await self.get_application() - # NB: request.json() is the first arg passed to the callable - request.json.return_value = asyncio.Future() - request.json.return_value.set_result(3) - with self.assertRaises(TimeoutError): - await sync_to_async_hail_query(request, time.sleep, timeout_s=1) - - with mock.patch('hail_search.web_app.ctypes.pythonapi.PyThreadState_SetAsyncExc') as mock_set_async_exc: - mock_set_async_exc.return_value = 2 - with self.assertRaises(SystemExit): - await sync_to_async_hail_query(request, time.sleep, timeout_s=1) - - async def test_status(self): - async with self.client.request('GET', '/status') as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertDictEqual(resp_json, {'success': True}) - - async def test_reload_globals(self): - async with self.client.request('POST', '/reload_globals') as resp: - resp_json = await resp.json() - self.assertTrue( - resp_json["('SNV_INDEL', 'GRCh38')"]['versions']['gnomad_genomes'], - ) - with mock.patch('hail_search.queries.base.hl.read_table') as mock_read_table: - mock_read_table.return_value = hl.Table.parallelize( - [], - hl.tstruct(), - globals=hl.Struct( - enums=hl.Struct(reloaded_enum=1), - versions=hl.Struct(reloaded_version=2), - ) - ) - async with self.client.request('POST', '/reload_globals') as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertDictEqual( - resp_json["('SNV_INDEL', 'GRCh38')"], - { - 'enums': {'reloaded_enum': 1}, - 'versions': {'reloaded_version': 2}, - }, - ) - - async def _assert_expected_search(self, results, gene_counts=None, **search_kwargs): - search_body = get_hail_search_body(**search_kwargs) - async with self.client.request('POST', '/search', json=search_body) as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertSetEqual(set(resp_json.keys()), {'results', 'total'}) - self.assertEqual(resp_json['total'], len(results)) - for i, result in enumerate(resp_json['results']): - self.assertEqual(result, results[i]) - - if gene_counts: - async with self.client.request('POST', '/gene_counts', json=search_body) as resp: - self.assertEqual(resp.status, 200) - gene_counts_json = await resp.json() - self.assertDictEqual(gene_counts_json, gene_counts) - - async def test_single_family_search(self): - variant_gene_counts = { - 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}} - } - await self._assert_expected_search( - [VARIANT1, VARIANT2, VARIANT3, VARIANT4], sample_data=FAMILY_2_VARIANT_SAMPLE_DATA, gene_counts=variant_gene_counts, - ) - - mito_gene_counts = { - 'ENSG00000210112': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000198886': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000198727': {'total': 1, 'families': {'F000002_2': 1}}, - } - await self._assert_expected_search( - [MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sample_data=FAMILY_2_MITO_SAMPLE_DATA, gene_counts=mito_gene_counts, - ) - - await self._assert_expected_search( - [GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], omit_data_type='SNV_INDEL', gene_counts=GCNV_GENE_COUNTS, - ) - - await self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4], sample_data=SV_WGS_SAMPLE_DATA, gene_counts=SV_GENE_COUNTS, - ) - - await self._assert_expected_search( - [VARIANT1, SV_VARIANT1, SV_VARIANT2, VARIANT2, VARIANT3, VARIANT4, SV_VARIANT3, GCNV_VARIANT1, SV_VARIANT4, - GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sample_data={ - 'SV_WES': EXPECTED_SAMPLE_DATA['SV_WES'], **FAMILY_2_ALL_SAMPLE_DATA, **SV_WGS_SAMPLE_DATA, - }, gene_counts={**variant_gene_counts, **mito_gene_counts, **GCNV_GENE_COUNTS, **SV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}}, - ) - - await self._assert_expected_search( - [GRCH37_VARIANT], genome_version='GRCh37', sample_data=FAMILY_2_VARIANT_SAMPLE_DATA) - - async def test_single_project_search(self): - variant_gene_counts = { - 'ENSG00000097046': {'total': 3, 'families': {'F000002_2': 2, 'F000003_3': 1}}, - 'ENSG00000177000': {'total': 3, 'families': {'F000002_2': 2, 'F000003_3': 1}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - } - await self._assert_expected_search( - [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4], omit_data_type='SV_WES', gene_counts=variant_gene_counts, - ) - - await self._assert_expected_search( - [GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], sample_data=SV_WES_SAMPLE_DATA, gene_counts={ - 'ENSG00000129562': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000013364': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000079616': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000103495': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000167371': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000280789': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000280893': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000281348': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000275023': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000277972': {'total': 1, 'families': {'F000002_2': 1}}, - } - ) - - await self._assert_expected_search( - [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - gene_counts={**variant_gene_counts, **GCNV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}} - ) - - async def test_multi_project_search(self): - await self._assert_expected_search( - [PROJECT_2_VARIANT, MULTI_PROJECT_VARIANT1, MULTI_PROJECT_VARIANT2, VARIANT3, VARIANT4], - gene_counts=GENE_COUNTS, sample_data=MULTI_PROJECT_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [PROJECT_2_VARIANT, MULTI_PROJECT_VARIANT1, SV_VARIANT1, SV_VARIANT2, MULTI_PROJECT_VARIANT2, VARIANT3, - VARIANT4, SV_VARIANT3, SV_VARIANT4], gene_counts={**GENE_COUNTS, **SV_GENE_COUNTS}, - sample_data={**MULTI_PROJECT_SAMPLE_DATA, **SV_WGS_SAMPLE_DATA}, - ) - - async def test_both_sample_types_search(self): - # One family (F000011_11) in a multi-project search has identical exome and genome data. - await self._assert_expected_search( - MULTI_PROJECT_BOTH_SAMPLE_TYPE_VARIANTS, gene_counts=GENE_COUNTS, sample_data=MULTI_PROJECT_SAMPLE_TYPES_SAMPLE_DATA, - ) - - # Variant 1 is de novo in exome but inherited and homozygous in genome. - # Variant 2 is inherited and homozygous in exome and de novo and homozygous in genome. - # Variant 3 is inherited in both sample types. - # Variant 4 is de novo in exome, but inherited in genome in the same parent that has variant 3. - inheritance_mode = 'recessive' - await self._assert_expected_search( - [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, [VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES]], - sample_data=FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA, inheritance_mode=inheritance_mode, - **COMP_HET_ALL_PASS_FILTERS - ) - await self._assert_expected_search( - [VARIANT1_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, VARIANT2_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, - [VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY]], - sample_data=FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA_MISSING_PARENTAL_WGS, inheritance_mode=inheritance_mode, - **COMP_HET_ALL_PASS_FILTERS - ) - - inheritance_mode = 'de_novo' - await self._assert_expected_search( - [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES], - sample_data=FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA, inheritance_mode=inheritance_mode, - ) - # Variant 2 fails inheritance when parental data is missing in genome - await self._assert_expected_search( - [VARIANT1_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY], - sample_data=FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA_MISSING_PARENTAL_WGS, inheritance_mode=inheritance_mode, - ) - - async def test_inheritance_filter(self): - inheritance_mode = 'any_affected' - await self._assert_expected_search( - [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - inheritance_mode=inheritance_mode, - ) - - await self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4], inheritance_mode=inheritance_mode, sample_data=SV_WGS_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [GCNV_VARIANT3], inheritance_mode=inheritance_mode, annotations=NEW_SV_FILTER, omit_data_type='SNV_INDEL', - ) - - await self._assert_expected_search( - [SV_VARIANT2], inheritance_mode=inheritance_mode, annotations=NEW_SV_FILTER, sample_data=SV_WGS_SAMPLE_DATA, - ) - - inheritance_mode = 'de_novo' - await self._assert_expected_search( - [VARIANT1, FAMILY_3_VARIANT, VARIANT4, GCNV_VARIANT1], inheritance_mode=inheritance_mode, - ) - - await self._assert_expected_search( - [SV_VARIANT1], inheritance_mode=inheritance_mode, sample_data=SV_WGS_SAMPLE_DATA, - ) - - inheritance_mode = 'x_linked_recessive' - await self._assert_expected_search([], inheritance_mode=inheritance_mode, sample_data=EXPECTED_SAMPLE_DATA_WITH_SEX) - await self._assert_expected_search([], inheritance_mode=inheritance_mode, sample_data=SV_WGS_SAMPLE_DATA_WITH_SEX) - - inheritance_mode = 'homozygous_recessive' - await self._assert_expected_search([VARIANT2, GCNV_VARIANT3], inheritance_mode=inheritance_mode) - - await self._assert_expected_search( - [PROJECT_2_VARIANT1, VARIANT2], inheritance_mode=inheritance_mode, sample_data=MULTI_PROJECT_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [SV_VARIANT4], inheritance_mode=inheritance_mode, sample_data=SV_WGS_SAMPLE_DATA, - ) - - gt_inheritance_filter = {'genotype': {'I000006_hg00733': 'ref_ref', 'I000005_hg00732': 'has_alt'}} - await self._assert_expected_search( - [VARIANT2], inheritance_filter=gt_inheritance_filter, sample_data=FAMILY_2_VARIANT_SAMPLE_DATA) - - inheritance_mode = 'compound_het' - await self._assert_expected_search( - [[VARIANT3, VARIANT4]], inheritance_mode=inheritance_mode, sample_data=MULTI_PROJECT_SAMPLE_DATA, gene_counts={ - 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 1, 'families': {'F000002_2': 1}}, - }, **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode=inheritance_mode, omit_data_type='SNV_INDEL', gene_counts={ - 'ENSG00000275023': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000277972': {'total': 1, 'families': {'F000002_2': 1}}, - }, **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], [GCNV_VARIANT3, GCNV_VARIANT4]], - inheritance_mode=inheritance_mode, gene_counts={ - 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000275023': {'total': 3, 'families': {'F000002_2': 3}}, - 'ENSG00000277258': {'total': 3, 'families': {'F000002_2': 3}}, - 'ENSG00000277972': {'total': 2, 'families': {'F000002_2': 2}}, - }, **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[SV_VARIANT1, SV_VARIANT2]], inheritance_mode=inheritance_mode, sample_data=SV_WGS_SAMPLE_DATA, - **COMP_HET_ALL_PASS_FILTERS, gene_counts={'ENSG00000171621': {'total': 2, 'families': {'F000011_11': 2}}}, - ) - - await self._assert_expected_search( - [[SV_VARIANT1, SV_VARIANT2], [VARIANT3, VARIANT4]], inheritance_mode=inheritance_mode, - sample_data={**SV_WGS_SAMPLE_DATA, **FAMILY_2_VARIANT_SAMPLE_DATA}, **COMP_HET_ALL_PASS_FILTERS, gene_counts={ - 'ENSG00000171621': {'total': 2, 'families': {'F000011_11': 2}}, - 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 1, 'families': {'F000002_2': 1}}, - }, - ) - - inheritance_mode = 'recessive' - await self._assert_expected_search( - [PROJECT_2_VARIANT1, VARIANT2, [VARIANT3, VARIANT4]], inheritance_mode=inheritance_mode, gene_counts={ - 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - }, sample_data=MULTI_PROJECT_SAMPLE_DATA, **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode=inheritance_mode, omit_data_type='SNV_INDEL', gene_counts={ - 'ENSG00000275023': {'total': 3, 'families': {'F000002_2': 3}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000277972': {'total': 1, 'families': {'F000002_2': 1}}, - }, **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], - inheritance_mode=inheritance_mode, gene_counts={ - 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 3, 'families': {'F000002_2': 3}}, - 'ENSG00000275023': {'total': 4, 'families': {'F000002_2': 4}}, - 'ENSG00000277258': {'total': 4, 'families': {'F000002_2': 4}}, - 'ENSG00000277972': {'total': 2, 'families': {'F000002_2': 2}}, - }, **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[SV_VARIANT1, SV_VARIANT2], SV_VARIANT4], inheritance_mode=inheritance_mode, sample_data=SV_WGS_SAMPLE_DATA, - **COMP_HET_ALL_PASS_FILTERS, gene_counts={ - 'ENSG00000171621': {'total': 2, 'families': {'F000011_11': 2}}, - 'ENSG00000184986': {'total': 1, 'families': {'F000011_11': 1}}, - } - ) - - async def test_quality_filter(self): - quality_filter = {'vcf_filter': 'pass'} - await self._assert_expected_search( - [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - quality_filter=quality_filter - ) - - await self._assert_expected_search( - [SV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2], quality_filter=quality_filter, - sample_data={**SV_WGS_SAMPLE_DATA, **FAMILY_2_MITO_SAMPLE_DATA} - ) - - await self._assert_expected_search( - [MITO_VARIANT1, MITO_VARIANT3], quality_filter={'min_gq': 60, 'min_hl': 5}, sample_data=FAMILY_2_MITO_SAMPLE_DATA, - ) - - gcnv_quality_filter = {'min_gq': 40, 'min_qs': 20} - await self._assert_expected_search( - [VARIANT2, MULTI_FAMILY_VARIANT, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT4], quality_filter=gcnv_quality_filter, - ) - - await self._assert_expected_search( - [], annotations=NEW_SV_FILTER, quality_filter=gcnv_quality_filter, omit_data_type='SNV_INDEL', - ) - - sv_quality_filter = {'min_gq_sv': 40} - await self._assert_expected_search( - [SV_VARIANT3, SV_VARIANT4], quality_filter=sv_quality_filter, sample_data=SV_WGS_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [], annotations=NEW_SV_FILTER, quality_filter=sv_quality_filter, sample_data=SV_WGS_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [VARIANT2, MULTI_FAMILY_VARIANT], quality_filter={'min_gq': 40, 'vcf_filter': 'pass'}, omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - quality_filter={'min_gq': 60, 'min_qs': 10, 'affected_only': True}, - ) - - await self._assert_expected_search( - [SV_VARIANT3, SV_VARIANT4], quality_filter={'min_gq_sv': 60, 'affected_only': True}, sample_data=SV_WGS_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [VARIANT1, VARIANT2, FAMILY_3_VARIANT], quality_filter={'min_ab': 50}, omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [VARIANT2, VARIANT3], quality_filter={'min_ab': 70, 'affected_only': True}, - omit_data_type='SV_WES', - ) - - quality_filter.update({'min_gq': 40, 'min_ab': 50}) - await self._assert_expected_search( - [VARIANT2, FAMILY_3_VARIANT], quality_filter=quality_filter, omit_data_type='SV_WES', - ) - - annotations = {'splice_ai': '0.0'} # Ensures no variants are filtered out by annotation/path filters - await self._assert_expected_search( - [VARIANT1, VARIANT2, FAMILY_3_VARIANT, MITO_VARIANT1, MITO_VARIANT3], quality_filter=quality_filter, omit_data_type='SV_WES', - annotations=annotations, pathogenicity={'clinvar': ['likely_pathogenic', 'vus_or_conflicting']}, - sample_data={**EXPECTED_SAMPLE_DATA, **FAMILY_2_MITO_SAMPLE_DATA}, - ) - - await self._assert_expected_search( - [VARIANT2, FAMILY_3_VARIANT], quality_filter=quality_filter, omit_data_type='SV_WES', - annotations=annotations, pathogenicity={'clinvar': ['pathogenic']}, - ) - - async def test_location_search(self): - await self._assert_expected_search( - [MULTI_FAMILY_VARIANT, VARIANT4], omit_data_type='SV_WES', **LOCATION_SEARCH, - ) - - # Test "large" gene list search - await self._assert_expected_search( - [VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4], omit_data_type='SV_WES', intervals=LOCATION_SEARCH['intervals'], - gene_ids=LOCATION_SEARCH['gene_ids'] + ['ENSG00000277258', 'ENSG00000275023'], - ) - - await self._assert_expected_search( - [GRCH37_VARIANT], intervals=[['7', 143268894, 143271480]], genome_version='GRCh37', sample_data=FAMILY_2_VARIANT_SAMPLE_DATA) - - sv_intervals = [['1', 9310023, 9380264], ['17', 38717636, 38724781]] - await self._assert_expected_search( - [GCNV_VARIANT3, GCNV_VARIANT4], intervals=sv_intervals, gene_ids=['ENSG00000275023'], omit_data_type='SNV_INDEL', - ) - - await self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT2], sample_data=SV_WGS_SAMPLE_DATA, intervals=sv_intervals, gene_ids=['ENSG00000171621'], - ) - - await self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT2, MULTI_PROJECT_GCNV_VARIANT3, GCNV_VARIANT4], intervals=sv_intervals, - sample_data={'SV_WES': EXPECTED_SAMPLE_DATA['SV_WES'] + SECOND_PROJECT_SV_WES_SAMPLE_DATA, **SV_WGS_SAMPLE_DATA}, - ) - - await self._assert_expected_search( - [VARIANT1, VARIANT2], omit_data_type='SV_WES', **EXCLUDE_LOCATION_SEARCH, - ) - - await self._assert_expected_search( - [GCNV_VARIANT1, GCNV_VARIANT2], intervals=sv_intervals, exclude_intervals=True, omit_data_type='SNV_INDEL', - ) - - await self._assert_expected_search( - [SV_VARIANT3, SV_VARIANT4], sample_data=SV_WGS_SAMPLE_DATA, intervals=sv_intervals, exclude_intervals=True, - ) - - await self._assert_expected_search( - [SELECTED_TRANSCRIPT_MULTI_FAMILY_VARIANT], omit_data_type='SV_WES', - intervals=LOCATION_SEARCH['intervals'][-1:], gene_ids=LOCATION_SEARCH['gene_ids'][1:] - ) - - await self._assert_expected_search( - [GCNV_VARIANT4], padded_interval={'chrom': '17', 'start': 38720781, 'end': 38738703, 'padding': 0.2}, - omit_data_type='SNV_INDEL', - ) - - await self._assert_expected_search( - [], padded_interval={'chrom': '17', 'start': 38720781, 'end': 38738703, 'padding': 0.1}, - omit_data_type='SNV_INDEL', - ) - - await self._assert_expected_search( - [SV_VARIANT4], padded_interval={'chrom': '14', 'start': 106692244, 'end': 106742587, 'padding': 0.1}, - sample_data=SV_WGS_SAMPLE_DATA, - ) - - # For gene search, return SVs annotated in gene even if they fall outside the gene interval - nearest_tss_gene_intervals = [['1', 9292894, 9369532]] - await self._assert_expected_search( - [SV_VARIANT1], sample_data=SV_WGS_SAMPLE_DATA, intervals=nearest_tss_gene_intervals, - ) - await self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT2], sample_data=SV_WGS_SAMPLE_DATA, intervals=nearest_tss_gene_intervals, - gene_ids=['ENSG00000171621'], - ) - - async def test_cluster_intervals(self): - intervals = [ - ['1', 11785723, 11806455], ['1', 91500851, 91525764], ['2', 1234, 5678], ['2', 12345, 67890], - ['7', 1, 11100], ['7', 202020, 20202020], - ] - - self.assertListEqual(BaseHailTableQuery.cluster_intervals(intervals, max_intervals=5), [ - ['1', 11785723, 11806455], ['1', 91500851, 91525764], ['2', 1234, 67890], - ['7', 1, 11100], ['7', 202020, 20202020], - ]) - - self.assertListEqual(BaseHailTableQuery.cluster_intervals(intervals, max_intervals=4), [ - ['1', 11785723, 11806455], ['1', 91500851, 91525764], ['2', 1234, 67890], ['7', 1, 20202020], - ]) - - self.assertListEqual(BaseHailTableQuery.cluster_intervals(intervals, max_intervals=3), [ - ['1', 11785723, 91525764], ['2', 1234, 67890], ['7', 1, 20202020], - ]) - - - async def test_variant_id_search(self): - await self._assert_expected_search([VARIANT2], omit_data_type='SV_WES', **RSID_SEARCH) - - await self._assert_expected_search([VARIANT1], omit_data_type='SV_WES', **VARIANT_ID_SEARCH) - - await self._assert_expected_search( - [VARIANT1], omit_data_type='SV_WES', variant_ids=VARIANT_ID_SEARCH['variant_ids'][:1], - ) - - await self._assert_expected_search( - [], omit_data_type='SV_WES', variant_ids=VARIANT_ID_SEARCH['variant_ids'][1:], - ) - - variant_keys = ['suffix_95340_DUP', 'suffix_140608_DUP'] - await self._assert_expected_search([GCNV_VARIANT1, GCNV_VARIANT4], omit_data_type='SNV_INDEL', variant_keys=variant_keys) - - await self._assert_expected_search([VARIANT1, GCNV_VARIANT1, GCNV_VARIANT4], variant_keys=variant_keys, **VARIANT_ID_SEARCH) - - await self._assert_expected_search([SV_VARIANT2, SV_VARIANT4], sample_data=SV_WGS_SAMPLE_DATA, variant_keys=[ - 'cohort_2911.chr1.final_cleanup_INS_chr1_160', 'phase2_DEL_chr14_4640', - ]) - - async def test_variant_lookup(self): - body = {'genome_version': 'GRCh38', 'variant_id': VARIANT_ID_SEARCH['variant_ids'][0]} - async with self.client.request('POST', '/lookup', json=body) as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertDictEqual(resp_json, VARIANT_LOOKUP_VARIANT) - - body['variant_id'] = VARIANT_ID_SEARCH['variant_ids'][1] - async with self.client.request('POST', '/lookup', json=body) as resp: - self.assertEqual(resp.status, 404) - - body.update({'genome_version': 'GRCh37', 'variant_id': ['7', 143270172, 'A', 'G']}) - async with self.client.request('POST', '/lookup', json=body) as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertDictEqual(resp_json, { - **{k: v for k, v in GRCH37_VARIANT.items() if k not in {'familyGuids', 'genotypes'}}, - 'familyGenotypes': {GRCH37_VARIANT['familyGuids'][0]: [ - {k: v for k, v in g.items() if k != 'individualGuid'} for g in GRCH37_VARIANT['genotypes'].values() - ]}, - }) - - body.update({'variant_id': ['M', 4429, 'G', 'A'], 'data_type': 'MITO', 'genome_version': 'GRCh38'}) - async with self.client.request('POST', '/lookup', json=body) as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertDictEqual(resp_json, { - **{k: v for k, v in MITO_VARIANT1.items() if k not in {'familyGuids', 'genotypes'}}, - 'familyGenotypes': {MITO_VARIANT1['familyGuids'][0]: [ - {k: v for k, v in g.items() if k != 'individualGuid'} for g in MITO_VARIANT1['genotypes'].values() - ]}, - }) - - body.update({'variant_id': 'phase2_DEL_chr14_4640', 'data_type': 'SV_WGS', 'sample_data': SV_WGS_SAMPLE_DATA['SV_WGS']}) - async with self.client.request('POST', '/lookup', json=body) as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertDictEqual(resp_json, SV_VARIANT4) - - body.update({'variant_id': 'suffix_140608_DUP', 'data_type': 'SV_WES', 'sample_data': EXPECTED_SAMPLE_DATA['SV_WES']}) - async with self.client.request('POST', '/lookup', json=body) as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertDictEqual(resp_json, { - **NO_GENOTYPE_GCNV_VARIANT, 'genotypes': { - individual: {k: v for k, v in genotype.items() if k not in {'start', 'end', 'numExon', 'geneIds'}} - for individual, genotype in GCNV_VARIANT4['genotypes'].items() - } - }) - - body['variant_id'] = 'suffix_140608_DEL' - async with self.client.request('POST', '/lookup', json=body) as resp: - self.assertEqual(resp.status, 404) - - async def test_multi_variant_lookup(self): - await self._test_multi_lookup(VARIANT_ID_SEARCH['variant_ids'], 'SNV_INDEL', [VARIANT1]) - - await self._test_multi_lookup([['7', 143270172, 'A', 'G']], 'SNV_INDEL', [GRCH37_VARIANT], genome_version='GRCh37') - - await self._test_multi_lookup([['M', 4429, 'G', 'A'], ['M', 14783, 'T', 'C']], 'MITO', [MITO_VARIANT1, MITO_VARIANT3]) - - await self._test_multi_lookup( - ['cohort_2911.chr1.final_cleanup_INS_chr1_160', 'phase2_DEL_chr14_4640'], - 'SV_WGS', [SV_VARIANT2, SV_VARIANT4], - ) - - await self._test_multi_lookup(['suffix_140608_DUP'], 'SV_WES', [NO_GENOTYPE_GCNV_VARIANT]) - - async def _test_multi_lookup(self, variant_ids, data_type, results, genome_version='GRCh38'): - body = {'genome_version': genome_version, 'data_type': data_type, 'variant_ids': variant_ids} - async with self.client.request('POST', '/multi_lookup', json=body) as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertDictEqual(resp_json, {'results': [ - {k: v for k, v in variant.items() if k not in {'familyGuids', 'genotypes'}} - for variant in results - ]}) - - async def test_frequency_filter(self): - sv_callset_filter = {'sv_callset': {'af': 0.05}} - # seqr af filter is ignored for SNV_INDEL - await self._assert_expected_search( - [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - frequencies={'seqr': {'af': 0.2}, **sv_callset_filter}, - ) - - await self._assert_expected_search( - [MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - frequencies={'seqr': {'ac': 5}, **sv_callset_filter}, - ) - - await self._assert_expected_search( - [MULTI_FAMILY_VARIANT, VARIANT4], frequencies={'seqr': {'ac': 4}}, omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [MULTI_FAMILY_VARIANT, VARIANT4], frequencies={'seqr': {'hh': 1}}, omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [VARIANT4], frequencies={'seqr': {'ac': 4, 'hh': 0}}, omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [MITO_VARIANT1, MITO_VARIANT2], frequencies={'seqr': {'af': 0.01}}, sample_data=FAMILY_2_MITO_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [SV_VARIANT1], frequencies=sv_callset_filter, sample_data=SV_WGS_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [VARIANT1, VARIANT2, VARIANT4], frequencies={'gnomad_genomes': {'af': 0.05}}, omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [VARIANT2, VARIANT4], frequencies={'gnomad_genomes': {'af': 0.05, 'hh': 1}}, omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [VARIANT2, VARIANT4, MITO_VARIANT1, MITO_VARIANT2], sample_data=FAMILY_2_ALL_SAMPLE_DATA, - frequencies={'gnomad_genomes': {'af': 0.005}, 'gnomad_mito': {'af': 0.05}}, - ) - - await self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT3, SV_VARIANT4], frequencies={'gnomad_svs': {'af': 0.001}}, sample_data=SV_WGS_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [VARIANT4], frequencies={'seqr': {'ac': 10}, 'gnomad_genomes': {'ac': 50}}, - omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4], frequencies={'seqr': {}, 'gnomad_genomes': {'af': None}}, - omit_data_type='SV_WES', - ) - - annotations = {'splice_ai': '0.0'} # Ensures no variants are filtered out by annotation/path filters - await self._assert_expected_search( - [VARIANT1, VARIANT2, VARIANT4], frequencies={'gnomad_genomes': {'af': 0.01, 'hh': 10}}, omit_data_type='SV_WES', - annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'likely_pathogenic', 'vus_or_conflicting']}, - ) - - await self._assert_expected_search( - [VARIANT2, VARIANT4], frequencies={'gnomad_genomes': {'af': 0.01}}, omit_data_type='SV_WES', - annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'vus_or_conflicting']}, - ) - - async def test_annotations_filter(self): - await self._assert_expected_search([VARIANT2], pathogenicity={'hgmd': ['hgmd_other']}, omit_data_type='SV_WES') - - pathogenicity = {'clinvar': ['likely_pathogenic', 'vus_or_conflicting', 'benign']} - await self._assert_expected_search( - [VARIANT1, VARIANT2, MITO_VARIANT1, MITO_VARIANT3], pathogenicity=pathogenicity, sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - exclude = {'clinvar': pathogenicity['clinvar'][1:]} - pathogenicity['clinvar'] = pathogenicity['clinvar'][:1] - annotations = {'SCREEN': ['CTCF-only', 'DNase-only'], 'UTRAnnotator': ['5_prime_UTR_stop_codon_loss_variant']} - selected_transcript_variant_2 = {**VARIANT2, 'selectedMainTranscriptId': 'ENST00000408919'} - await self._assert_expected_search( - [VARIANT1, selected_transcript_variant_2, VARIANT4, MITO_VARIANT3], pathogenicity=pathogenicity, annotations=annotations, - sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [VARIANT1, VARIANT4, MITO_VARIANT3], exclude=exclude, pathogenicity=pathogenicity, - annotations=annotations, sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [], pathogenicity=pathogenicity, annotations=annotations, sample_data=FAMILY_2_VARIANT_SAMPLE_DATA, - genome_version='GRCh37', - ) - - annotations = { - 'missense': ['missense_variant'], 'in_frame': ['inframe_insertion', 'inframe_deletion'], 'frameshift': None, - 'structural_consequence': ['INTRONIC', 'LOF'], - } - await self._assert_expected_search( - [VARIANT1, VARIANT2, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4, MITO_VARIANT2, MITO_VARIANT3], pathogenicity=pathogenicity, - annotations=annotations, sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [VARIANT2, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4, GCNV_VARIANT3, GCNV_VARIANT4], annotations=annotations, - ) - - await self._assert_expected_search([SV_VARIANT1], annotations=annotations, sample_data=SV_WGS_SAMPLE_DATA) - - annotations['splice_ai'] = '0.005' - annotations['structural'] = ['gCNV_DUP', 'DEL'] - await self._assert_expected_search( - [VARIANT2, MULTI_FAMILY_VARIANT, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - annotations=annotations, - ) - - await self._assert_expected_search([SV_VARIANT1, SV_VARIANT4], annotations=annotations, sample_data=SV_WGS_SAMPLE_DATA) - - annotations = {'other': ['non_coding_transcript_exon_variant__canonical', 'non_coding_transcript_exon_variant']} - await self._assert_expected_search( - [VARIANT1, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_3, MITO_VARIANT1, MITO_VARIANT3], - pathogenicity=pathogenicity, annotations=annotations, sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2], - gene_ids=LOCATION_SEARCH['gene_ids'][1:], annotations=annotations, omit_data_type='SV_WES', - ) - - annotations['other'].append('intron_variant') - await self._assert_expected_search( - [SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2, SELECTED_TRANSCRIPT_MULTI_FAMILY_VARIANT], - gene_ids=LOCATION_SEARCH['gene_ids'][1:], annotations=annotations, omit_data_type='SV_WES', - ) - - annotations['other'] = annotations['other'][:1] - annotations['splice_ai'] = '0.005' - await self._assert_expected_search( - [VARIANT1, VARIANT3, MITO_VARIANT1, MITO_VARIANT3], - pathogenicity=pathogenicity, annotations=annotations, sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - annotations['extended_splice_site'] = ['extended_intronic_splice_region_variant'] - await self._assert_expected_search( - [VARIANT1, VARIANT3, VARIANT4, MITO_VARIANT1, MITO_VARIANT3], - pathogenicity=pathogenicity, annotations=annotations, sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - annotations = {'motif_feature': ['TF_binding_site_variant'], 'regulatory_feature': ['regulatory_region_variant']} - await self._assert_expected_search( - [VARIANT3, VARIANT4], annotations=annotations, sample_data=FAMILY_2_VARIANT_SAMPLE_DATA, - ) - - async def test_secondary_annotations_filter(self): - annotations_1 = {'missense': ['missense_variant']} - annotations_2 = {'other': ['intron_variant']} - - await self._assert_expected_search( - [[VARIANT3, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4]], inheritance_mode='compound_het', omit_data_type='SV_WES', - annotations=annotations_1, annotations_secondary=annotations_2, - ) - - await self._assert_expected_search( - [VARIANT2, [VARIANT3, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4]], inheritance_mode='recessive', omit_data_type='SV_WES', - annotations=annotations_1, annotations_secondary=annotations_2, - ) - - await self._assert_expected_search( - [[VARIANT3, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4]], inheritance_mode='recessive', omit_data_type='SV_WES', - annotations=annotations_2, annotations_secondary=annotations_1, - ) - - gcnv_annotations_1 = {'structural': ['gCNV_DUP']} - gcnv_annotations_2 = {'structural_consequence': ['LOF'], 'structural': []} - - await self._assert_expected_search( - [[GCNV_VARIANT3, GCNV_VARIANT4]], omit_data_type='SNV_INDEL', inheritance_mode='compound_het', - annotations=gcnv_annotations_1, annotations_secondary=gcnv_annotations_2, - ) - - await self._assert_expected_search( - [GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], omit_data_type='SNV_INDEL', inheritance_mode='recessive', - annotations=gcnv_annotations_2, annotations_secondary=gcnv_annotations_1, - ) - - # Do not return pairs where annotations match in a non-paired gene - gcnv_annotations_no_pair = {'structural_consequence': ['COPY_GAIN']} - await self._assert_expected_search( - [], omit_data_type='SNV_INDEL', inheritance_mode='compound_het', - annotations=gcnv_annotations_1, annotations_secondary=gcnv_annotations_no_pair, - ) - - await self._assert_expected_search( - [], omit_data_type='SNV_INDEL', inheritance_mode='compound_het', - annotations={**gcnv_annotations_1, **gcnv_annotations_no_pair}, - ) - - await self._assert_expected_search( - [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4]], inheritance_mode='compound_het', - annotations=annotations_1, annotations_secondary=gcnv_annotations_2, - ) - - await self._assert_expected_search( - [VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4], GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], - inheritance_mode='recessive', - annotations={**annotations_1, **gcnv_annotations_1}, annotations_secondary={**annotations_2, **gcnv_annotations_2}, - ) - - await self._assert_expected_search( - [VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], MULTI_PROJECT_GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], - inheritance_mode='recessive', - annotations={**annotations_1, **gcnv_annotations_1}, annotations_secondary={**annotations_2, **gcnv_annotations_2}, - intervals=[['1', 38717636, 38724781], ['17', 38717636, 38724781]], - sample_data={**EXPECTED_SAMPLE_DATA, 'SV_WES': EXPECTED_SAMPLE_DATA['SV_WES'] + SECOND_PROJECT_SV_WES_SAMPLE_DATA}, - ) - - await self._assert_expected_search( - [MULTI_DATA_TYPE_COMP_HET_VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [GCNV_VARIANT3, GCNV_VARIANT4]], - inheritance_mode='recessive', - annotations={**annotations_1, 'structural': ['gCNV_DEL'], 'structural_consequence': ['INTRONIC']}, - annotations_secondary={**annotations_2, **gcnv_annotations_1}, - gene_ids=['ENSG00000277258', 'ENSG00000275023'], intervals=[['1', 38717636, 38724781], ['17', 38717636, 38724781]], - ) - - await self._assert_expected_search( - [MULTI_DATA_TYPE_COMP_HET_VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4]], - inheritance_mode='recessive', annotations={**annotations_1, 'structural': [], 'structural_consequence': []}, - annotations_secondary=gcnv_annotations_2, - gene_ids=['ENSG00000277258', 'ENSG00000275023'], intervals=[['1', 38717636, 38724781], ['17', 38717636, 38724781]], - ) - - sv_annotations_1 = {'structural': ['INS', 'LOF']} - sv_annotations_2 = {'structural': ['DEL', 'gCNV_DUP'], 'structural_consequence': ['INTRONIC']} - - await self._assert_expected_search( - [[SV_VARIANT1, SV_VARIANT2]], sample_data=SV_WGS_SAMPLE_DATA, inheritance_mode='compound_het', - annotations=sv_annotations_1, annotations_secondary=sv_annotations_2, - ) - - await self._assert_expected_search( - [[SV_VARIANT1, SV_VARIANT2], SV_VARIANT4], sample_data=SV_WGS_SAMPLE_DATA, inheritance_mode='recessive', - annotations=sv_annotations_2, annotations_secondary=sv_annotations_1, - ) - - pathogenicity = {'clinvar': ['likely_pathogenic', 'vus_or_conflicting']} - await self._assert_expected_search( - [VARIANT2, [VARIANT3, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4]], inheritance_mode='recessive', omit_data_type='SV_WES', - annotations=annotations_2, annotations_secondary=annotations_1, pathogenicity=pathogenicity, - ) - - await self._assert_expected_search( - [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [GCNV_VARIANT3, GCNV_VARIANT4]], - inheritance_mode='compound_het', pathogenicity=pathogenicity, - annotations=gcnv_annotations_2, annotations_secondary=gcnv_annotations_1, - ) - - await self._assert_expected_search( - [VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], - inheritance_mode='recessive', pathogenicity=pathogenicity, - annotations=gcnv_annotations_2, annotations_secondary=gcnv_annotations_1, - ) - - selected_transcript_annotations = {'other': ['non_coding_transcript_exon_variant']} - await self._assert_expected_search( - [VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], GCNV_VARIANT3], - inheritance_mode='recessive', pathogenicity=pathogenicity, - annotations=gcnv_annotations_2, annotations_secondary=selected_transcript_annotations, - ) - - # Search works with a different number of samples within the family - missing_gt_gcnv_variant = { - **GCNV_VARIANT4, 'genotypes': {k: v for k, v in GCNV_VARIANT4['genotypes'].items() if k != 'I000005_hg00732'} - } - await self._assert_expected_search( - [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, missing_gt_gcnv_variant]], - inheritance_mode='compound_het', pathogenicity=pathogenicity, - annotations=gcnv_annotations_2, annotations_secondary=selected_transcript_annotations, - sample_data={**EXPECTED_SAMPLE_DATA, 'SV_WES': SV_WES_SAMPLE_DATA['SV_WES'][:1] + SV_WES_SAMPLE_DATA['SV_WES'][2:]} - - ) - - # Do not return pairs where annotations match in a non-paired gene - await self._assert_expected_search( - [GCNV_VARIANT3], inheritance_mode='recessive', - annotations=gcnv_annotations_2, annotations_secondary=selected_transcript_annotations, - ) - - screen_annotations = {'SCREEN': ['CTCF-only']} - await self._assert_expected_search( - [], inheritance_mode='recessive', omit_data_type='SV_WES', - annotations=screen_annotations, annotations_secondary=annotations_1, - ) - - await self._assert_expected_search( - [[VARIANT3, VARIANT4]], inheritance_mode='recessive', omit_data_type='SV_WES', - annotations=screen_annotations, annotations_secondary=annotations_2, - ) - - await self._assert_expected_search( - [VARIANT2, [SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_3, VARIANT4]], inheritance_mode='recessive', - annotations=screen_annotations, annotations_secondary=selected_transcript_annotations, - pathogenicity=pathogenicity, omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2, [VARIANT3, VARIANT4]], - annotations={**selected_transcript_annotations, **screen_annotations}, annotations_secondary=annotations_2, - inheritance_mode='recessive', omit_data_type='SV_WES', - ) - - async def test_in_silico_filter(self): - in_silico = {'eigen': '3.5', 'mut_taster': 'N', 'vest': 0.5} - await self._assert_expected_search( - [VARIANT1, VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], in_silico=in_silico, - sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [GRCH37_VARIANT], genome_version='GRCh37', in_silico=in_silico, - sample_data=FAMILY_2_VARIANT_SAMPLE_DATA, - ) - - in_silico['requireScore'] = True - in_silico.pop('eigen') - await self._assert_expected_search( - [VARIANT4, MITO_VARIANT2], in_silico=in_silico, sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - sv_in_silico = {'strvctvre': 0.1, 'requireScore': True} - await self._assert_expected_search( - [GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], omit_data_type='SNV_INDEL', in_silico=sv_in_silico, - ) - - await self._assert_expected_search( - [SV_VARIANT4], sample_data=SV_WGS_SAMPLE_DATA, in_silico=sv_in_silico, - ) - - async def test_search_errors(self): - search_body = get_hail_search_body(sample_data=FAMILY_2_MISSING_SAMPLE_DATA) - async with self.client.request('POST', '/search', json=search_body) as resp: - self.assertEqual(resp.status, 400) - reason = resp.reason - self.assertEqual(reason, 'The following samples are available in seqr but missing the loaded data: NA19675_1, NA19678') - - search_body = get_hail_search_body(sample_data=MULTI_PROJECT_MISSING_SAMPLE_DATA) - async with self.client.request('POST', '/search', json=search_body) as resp: - self.assertEqual(resp.status, 400) - reason = resp.reason - self.assertEqual(reason, 'The following samples are available in seqr but missing the loaded data: NA19675_1, NA19678') - - search_body = get_hail_search_body( - intervals=LOCATION_SEARCH['intervals'] + [['1', 1, 999999999]], omit_data_type='SV_WES', - ) - async with self.client.request('POST', '/search', json=search_body) as resp: - self.assertEqual(resp.status, 400) - reason = resp.reason - self.assertEqual(reason, 'Invalid intervals: 1:1-999999999') - - async def test_sort(self): - await self._assert_expected_search( - [_sorted(VARIANT4, [2, 2]), _sorted(MITO_VARIANT2, [11, 11]), _sorted(VARIANT2, [12, 12]), - _sorted(MITO_VARIANT3, [17, 17]), _sorted(MITO_VARIANT1, [22, 22]), _sorted(VARIANT3, [26, 27]), - _sorted(VARIANT1, [None, None])], sample_data=FAMILY_2_ALL_SAMPLE_DATA, sort='protein_consequence', - ) - - await self._assert_expected_search( - [_sorted(GCNV_VARIANT2, [0]), _sorted(GCNV_VARIANT3, [0]), _sorted(GCNV_VARIANT4, [0]), - _sorted(GCNV_VARIANT1, [3])], omit_data_type='SNV_INDEL', sort='protein_consequence', - ) - - await self._assert_expected_search( - [_sorted(VARIANT4, [2, 2]), _sorted(GCNV_VARIANT2, [4.5, 0]), _sorted(GCNV_VARIANT3, [4.5, 0]), _sorted(GCNV_VARIANT4, [4.5, 0]), - _sorted(GCNV_VARIANT1, [4.5, 3]), _sorted(VARIANT2, [12, 12]), - _sorted(MULTI_FAMILY_VARIANT, [26, 27]), _sorted(VARIANT1, [None, None])], sort='protein_consequence', - ) - - await self._assert_expected_search( - [_sorted(SV_VARIANT1, [11]), _sorted(SV_VARIANT2, [12]), _sorted(SV_VARIANT3, [12]), _sorted(SV_VARIANT4, [12])], - sample_data=SV_WGS_SAMPLE_DATA, sort='protein_consequence', - ) - - await self._assert_expected_search( - [_sorted(VARIANT4, [2, 2]), _sorted(SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2, [12, 26]), - _sorted(SELECTED_ANNOTATION_TRANSCRIPT_MULTI_FAMILY_VARIANT, [26, 26])], - omit_data_type='SV_WES', sort='protein_consequence', - annotations={'other': ['non_coding_transcript_exon_variant'], 'splice_ai': '0'}, - ) - - await self._assert_expected_search( - [_sorted(VARIANT1, [4]), _sorted(VARIANT2, [8]), _sorted(MULTI_FAMILY_VARIANT, [12.5]), - _sorted(VARIANT4, [12.5]), GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], sort='pathogenicity', - ) - - await self._assert_expected_search( - [ _sorted(MITO_VARIANT3, [4]), _sorted(VARIANT1, [4, None]), _sorted(VARIANT2, [8, 3]), - _sorted(MITO_VARIANT1, [11]), _sorted(MITO_VARIANT2, [12.5]), _sorted(VARIANT3, [12.5, None]), - _sorted(VARIANT4, [12.5, None])], - sort='pathogenicity_hgmd', sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [_sorted(VARIANT2, [0]), _sorted(MITO_VARIANT1, [0]), _sorted(MITO_VARIANT2, [0]), - _sorted(VARIANT4, [0.00026519427774474025]), _sorted(VARIANT1, [0.034449315071105957]), - _sorted(MITO_VARIANT3, [0.05534649267792702]), _sorted(VARIANT3, [0.38041073083877563])], - sort='gnomad', sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [_sorted(VARIANT1, [0]), _sorted(MULTI_FAMILY_VARIANT, [0]), _sorted(VARIANT4, [0]), - _sorted(VARIANT2, [0.28899794816970825]), GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - sort='gnomad_exomes', - ) - - await self._assert_expected_search( - [_sorted(VARIANT4, [2]),_sorted(MULTI_FAMILY_VARIANT, [4]), _sorted(VARIANT1, [9]), - _sorted(VARIANT2, [28]), _sorted(GCNV_VARIANT3, [35]), _sorted(GCNV_VARIANT4, [115]), - _sorted(GCNV_VARIANT2, [284]), _sorted(GCNV_VARIANT1, [1763])], - sort='callset_af', - ) - - await self._assert_expected_search( - [_sorted(MITO_VARIANT1, [0]), _sorted(MITO_VARIANT2, [0]), _sorted(VARIANT4, [2]), - _sorted(MITO_VARIANT3, [3]), _sorted(VARIANT3, [4]), _sorted(VARIANT1, [9]), - _sorted(VARIANT2, [28])], - sort='callset_af', sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [_sorted(VARIANT4, [-29.899999618530273]), _sorted(VARIANT2, [-20.899999618530273]), - _sorted(VARIANT1, [-4.668000221252441]), _sorted(MULTI_FAMILY_VARIANT, [-2.753999948501587]), - GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], sort='cadd', - ) - - await self._assert_expected_search( - [_sorted(VARIANT4, [-0.5260000228881836]), _sorted(VARIANT2, [-0.19699999690055847]), - _sorted(VARIANT1, [0]), _sorted(MULTI_FAMILY_VARIANT, [0])], omit_data_type='SV_WES', sort='revel', - ) - - await self._assert_expected_search( - [_sorted(MULTI_FAMILY_VARIANT, [-0.009999999776482582]), _sorted(VARIANT2, [0]), _sorted(VARIANT4, [0]), - _sorted(VARIANT1, [0])], omit_data_type='SV_WES', sort='splice_ai', - ) - - await self._assert_expected_search( - [_sorted(VARIANT2, [-0.9977999925613403, -0.9977999925613403]), _sorted(VARIANT1, [0, 0]), - _sorted(MULTI_FAMILY_VARIANT, [0, 0]), _sorted(VARIANT4, [0, 0])], omit_data_type='SV_WES', sort='alphamissense', - ) - - sort = 'in_omim' - await self._assert_expected_search( - [_sorted(MULTI_FAMILY_VARIANT, [0, -2]), _sorted(VARIANT2, [0, -1]), _sorted(VARIANT4, [0, -1]), _sorted(VARIANT1, [1, 0])], - omit_data_type='SV_WES', sort=sort, sort_metadata=OMIM_SORT_METADATA, - ) - - await self._assert_expected_search( - [_sorted(GCNV_VARIANT3, [-1]), _sorted(GCNV_VARIANT4, [-1]), _sorted(GCNV_VARIANT1, [0]), _sorted(GCNV_VARIANT2, [0])], - omit_data_type='SNV_INDEL', sort=sort, sort_metadata=OMIM_SORT_METADATA, - ) - - await self._assert_expected_search( - [_sorted(MULTI_FAMILY_VARIANT, [0, -2]), _sorted(VARIANT2, [0, -1]), _sorted(VARIANT4, [0, -1]), - _sorted(GCNV_VARIANT3, [0, -1]), _sorted(GCNV_VARIANT4, [0, -1]), _sorted(GCNV_VARIANT1, [0, 0]), - _sorted(GCNV_VARIANT2, [0, 0]), _sorted(VARIANT1, [1, 0])], sort=sort, sort_metadata=OMIM_SORT_METADATA, - ) - - await self._assert_expected_search( - [_sorted(VARIANT2, [0, -1]), _sorted(MULTI_FAMILY_VARIANT, [1, -1]), _sorted(VARIANT1, [1, 0]), _sorted(VARIANT4, [1, 0])], - omit_data_type='SV_WES', sort=sort, sort_metadata=['ENSG00000177000'], - ) - - constraint_sort_metadata = {'ENSG00000177000': 2, 'ENSG00000275023': 3, 'ENSG00000097046': 4} - sort = 'constraint' - await self._assert_expected_search( - [_sorted(VARIANT2, [2, 2]), _sorted(MULTI_FAMILY_VARIANT, [4, 2]), _sorted(VARIANT4, [4, 4]), - _sorted(VARIANT1, [None, None])], omit_data_type='SV_WES', sort=sort, sort_metadata=constraint_sort_metadata, - ) - - await self._assert_expected_search( - [_sorted(GCNV_VARIANT3, [3]), _sorted(GCNV_VARIANT4, [3]), _sorted(GCNV_VARIANT1, [None]), - _sorted(GCNV_VARIANT2, [None])], omit_data_type='SNV_INDEL', sort=sort, sort_metadata=constraint_sort_metadata, - ) - - await self._assert_expected_search( - [_sorted(VARIANT2, [2, 2]), _sorted(GCNV_VARIANT3, [3, 3]), _sorted(GCNV_VARIANT4, [3, 3]), - _sorted(MULTI_FAMILY_VARIANT, [4, 2]), _sorted(VARIANT4, [4, 4]), _sorted(VARIANT1, [None, None]), - _sorted(GCNV_VARIANT1, [None, None]), _sorted(GCNV_VARIANT2, [None, None])], - sort=sort, sort_metadata=constraint_sort_metadata, - ) - - await self._assert_expected_search( - [_sorted(VARIANT2, [3, 3]), _sorted(MULTI_FAMILY_VARIANT, [None, 3]), _sorted(VARIANT1, [None, None]), - _sorted(VARIANT4, [None, None])], omit_data_type='SV_WES', sort='prioritized_gene', - sort_metadata={'ENSG00000177000': 3}, - ) - - await self._assert_expected_search( - [_sorted(MULTI_PROJECT_VARIANT1, [2]), _sorted(MULTI_PROJECT_VARIANT2, [2]), - _sorted(VARIANT3, [2]), _sorted(VARIANT4, [2]), _sorted(PROJECT_2_VARIANT, [11])], - sort='family_guid', sample_data=MULTI_PROJECT_SAMPLE_DATA, - ) - - # size sort only applies to SVs, so has no impact on other variant - await self._assert_expected_search( - [_sorted(GCNV_VARIANT1, [-171766]), _sorted(GCNV_VARIANT2, [-17768]), _sorted(GCNV_VARIANT4, [-14487]), - _sorted(GCNV_VARIANT3, [-2666]), VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4], sort='size', - ) - - await self._assert_expected_search( - [_sorted(SV_VARIANT4, [-46343]), _sorted(SV_VARIANT1, [-104]), _sorted(SV_VARIANT2, [-50]), - _sorted(SV_VARIANT3, [-50])], sample_data=SV_WGS_SAMPLE_DATA, sort='size', - ) - - # sort applies to compound hets - await self._assert_expected_search( - [[_sorted(VARIANT4, [-0.5260000228881836]), _sorted(VARIANT3, [0])], - _sorted(VARIANT2, [-0.19699999690055847])], - sort='revel', inheritance_mode='recessive', omit_data_type='SV_WES', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(VARIANT3, [-0.009999999776482582]), _sorted(VARIANT4, [0])], _sorted(VARIANT2, [0])], - sort='splice_ai', inheritance_mode='recessive', omit_data_type='SV_WES', **COMP_HET_ALL_PASS_FILTERS, - ) - - async def test_multi_data_type_comp_het_sort(self): - await self._assert_expected_search( - [[_sorted(VARIANT4, [2, 2]), _sorted(VARIANT3, [26, 27])], - _sorted(GCNV_VARIANT3, [4.5, 0]), [_sorted(GCNV_VARIANT3, [0]), _sorted(GCNV_VARIANT4, [0])], - [_sorted(GCNV_VARIANT4, [4.5, 0]), _sorted(MULTI_DATA_TYPE_COMP_HET_VARIANT2, [12, 12])], - _sorted(VARIANT2, [12, 12])], - sort='protein_consequence', inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(GCNV_VARIANT4, [-14487]), _sorted(GCNV_VARIANT3, [-2666])], - [_sorted(GCNV_VARIANT4, [-14487]), MULTI_DATA_TYPE_COMP_HET_VARIANT2], - [VARIANT3, VARIANT4]], - sort='size', inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(MULTI_DATA_TYPE_COMP_HET_VARIANT2, [8]), GCNV_VARIANT4], - [_sorted(VARIANT3, [12.5]), _sorted(VARIANT4, [12.5])], - [GCNV_VARIANT3, GCNV_VARIANT4]], - sort='pathogenicity', inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(VARIANT4, [-0.6869999766349792]), _sorted(VARIANT3, [0])], _sorted(VARIANT2, [0]), - [_sorted(MULTI_DATA_TYPE_COMP_HET_VARIANT2, [0]), GCNV_VARIANT4], - GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], - sort='mut_pred', inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(VARIANT3, [-0.009999999776482582]), _sorted(VARIANT4, [0])], - [_sorted(MULTI_DATA_TYPE_COMP_HET_VARIANT2, [0]), GCNV_VARIANT4], - [GCNV_VARIANT3, GCNV_VARIANT4]], - sort='splice_ai', inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(GCNV_VARIANT3, [-0.7860000133514404]), _sorted(GCNV_VARIANT4, [-0.7099999785423279])], - [_sorted(GCNV_VARIANT4, [-0.7099999785423279]), MULTI_DATA_TYPE_COMP_HET_VARIANT2], - [VARIANT3, VARIANT4]], - sort='strvctvre', inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(VARIANT4, [2]), _sorted(VARIANT3, [4])], - [_sorted(MULTI_DATA_TYPE_COMP_HET_VARIANT2, [28]), _sorted(GCNV_VARIANT4, [115])], - [_sorted(GCNV_VARIANT3, [35]), _sorted(GCNV_VARIANT4, [115])]], - sort='callset_af', inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(VARIANT3, [0]), _sorted(VARIANT4, [0])], - [_sorted(MULTI_DATA_TYPE_COMP_HET_VARIANT2, [0.28899794816970825]), GCNV_VARIANT4], - [GCNV_VARIANT3, GCNV_VARIANT4]], - sort='gnomad_exomes', inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(VARIANT3, [0, -2]), _sorted(VARIANT4, [0, -1])], - [_sorted(GCNV_VARIANT3, [-1]), _sorted(GCNV_VARIANT4, [-1])], - [_sorted(GCNV_VARIANT4, [0, -1]), _sorted(MULTI_DATA_TYPE_COMP_HET_VARIANT2, [1, -1])]], - sort='in_omim', sort_metadata=OMIM_SORT_METADATA, inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, - ) diff --git a/hail_search/test_utils.py b/hail_search/test_utils.py deleted file mode 100644 index 6fc725dcac..0000000000 --- a/hail_search/test_utils.py +++ /dev/null @@ -1,1177 +0,0 @@ -from copy import deepcopy - - -FAMILY_3_SAMPLE = { - 'sample_id': 'NA20870', 'individual_guid': 'I000007_na20870', 'family_guid': 'F000003_3', - 'project_guid': 'R0001_1kg', 'affected': 'A', 'sample_type': 'WES', -} -FAMILY_2_VARIANT_SAMPLE_DATA_WITH_SEX = {'SNV_INDEL': [ - {'sample_id': 'HG00731', 'individual_guid': 'I000004_hg00731', 'family_guid': 'F000002_2', 'project_guid': 'R0001_1kg', 'affected': 'A', 'sample_type': 'WES', 'is_male': False}, - {'sample_id': 'HG00732', 'individual_guid': 'I000005_hg00732', 'family_guid': 'F000002_2', 'project_guid': 'R0001_1kg', 'affected': 'N', 'sample_type': 'WES', 'is_male': True}, - {'sample_id': 'HG00733', 'individual_guid': 'I000006_hg00733', 'family_guid': 'F000002_2', 'project_guid': 'R0001_1kg', 'affected': 'N', 'sample_type': 'WES', 'is_male': False}, -]} -FAMILY_2_VARIANT_SAMPLE_DATA = deepcopy(FAMILY_2_VARIANT_SAMPLE_DATA_WITH_SEX) -for s in FAMILY_2_VARIANT_SAMPLE_DATA['SNV_INDEL']: - s.pop('is_male') - -EXPECTED_SAMPLE_DATA_WITH_SEX = { - 'SV_WES': [ - {'sample_id': 'HG00731', 'individual_guid': 'I000004_hg00731', 'family_guid': 'F000002_2', 'project_guid': 'R0001_1kg', 'affected': 'A', 'sample_type': 'WES', 'is_male': False}, - {'sample_id': 'HG00732', 'individual_guid': 'I000005_hg00732', 'family_guid': 'F000002_2', 'project_guid': 'R0001_1kg', 'affected': 'N', 'sample_type': 'WES', 'is_male': True}, - {'sample_id': 'HG00733', 'individual_guid': 'I000006_hg00733', 'family_guid': 'F000002_2', 'project_guid': 'R0001_1kg', 'affected': 'N', 'sample_type': 'WES', 'is_male': False} - ], -} -EXPECTED_SAMPLE_DATA_WITH_SEX.update(FAMILY_2_VARIANT_SAMPLE_DATA_WITH_SEX) -EXPECTED_SAMPLE_DATA_WITH_SEX['SNV_INDEL'].append({'is_male': True, **FAMILY_3_SAMPLE}) - -EXPECTED_SAMPLE_DATA = deepcopy(EXPECTED_SAMPLE_DATA_WITH_SEX) -for samples in EXPECTED_SAMPLE_DATA.values(): - for s in samples: - s.pop('is_male') - -CUSTOM_AFFECTED_SAMPLE_DATA = {'SNV_INDEL': deepcopy(EXPECTED_SAMPLE_DATA['SNV_INDEL'])} -CUSTOM_AFFECTED_SAMPLE_DATA['SNV_INDEL'][0]['affected'] = 'N' -CUSTOM_AFFECTED_SAMPLE_DATA['SNV_INDEL'][1]['affected'] = 'A' -CUSTOM_AFFECTED_SAMPLE_DATA['SNV_INDEL'][2]['affected'] = 'U' - -FAMILY_1_SAMPLE_DATA = { - 'SNV_INDEL': [ - {'sample_id': 'NA19675_1', 'individual_guid': 'I000001_na19675', 'family_guid': 'F000001_1', 'project_guid': 'R0001_1kg', 'sample_type': 'WES', 'affected': 'A'}, - {'sample_id': 'NA19678', 'individual_guid': 'I000002_na19678', 'family_guid': 'F000001_1', 'project_guid': 'R0001_1kg', 'sample_type': 'WES', 'affected': 'N'}, - ], -} -FAMILY_2_MISSING_SAMPLE_DATA = deepcopy(FAMILY_1_SAMPLE_DATA) -for s in FAMILY_2_MISSING_SAMPLE_DATA['SNV_INDEL']: - s['family_guid'] = 'F000002_2' - -FAMILY_2_MITO_SAMPLE_DATA = {'MITO': [ - {'sample_id': 'HG00733', 'individual_guid': 'I000006_hg00733', 'family_guid': 'F000002_2', 'project_guid': 'R0001_1kg', 'affected': 'N', 'sample_type': 'WES'}, -]} -FAMILY_2_ALL_SAMPLE_DATA = deepcopy(FAMILY_2_VARIANT_SAMPLE_DATA) -FAMILY_2_ALL_SAMPLE_DATA.update(FAMILY_2_MITO_SAMPLE_DATA) - -ALL_AFFECTED_SAMPLE_DATA = deepcopy(EXPECTED_SAMPLE_DATA) -ALL_AFFECTED_SAMPLE_DATA.update(FAMILY_2_MITO_SAMPLE_DATA) -FAMILY_5_SAMPLE = { - 'sample_id': 'NA20874', 'individual_guid': 'I000009_na20874', 'family_guid': 'F000005_5', 'project_guid': 'R0001_1kg', 'affected': 'N', 'sample_type': 'WES', -} -ALL_AFFECTED_SAMPLE_DATA['SNV_INDEL'].append(FAMILY_5_SAMPLE) -FAMILY_11_SAMPLE_WGS = { - 'sample_id': 'NA20885', 'individual_guid': 'I000015_na20885', 'family_guid': 'F000011_11', 'project_guid': 'R0003_test', 'affected': 'A', 'sample_type': 'WGS', -} -MULTI_PROJECT_SAMPLE_DATA = deepcopy(FAMILY_2_VARIANT_SAMPLE_DATA) -MULTI_PROJECT_SAMPLE_DATA['SNV_INDEL'].append({**FAMILY_11_SAMPLE_WGS, 'sample_type': 'WES'}) - -FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA = deepcopy(FAMILY_2_VARIANT_SAMPLE_DATA) -FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA['SNV_INDEL'].extend([ - {**s, 'sample_type': 'WGS'} for s in FAMILY_2_VARIANT_SAMPLE_DATA['SNV_INDEL']] -) - -FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA_MISSING_PARENTAL_WGS = deepcopy(FAMILY_2_VARIANT_SAMPLE_DATA) -FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA_MISSING_PARENTAL_WGS['SNV_INDEL'].extend([ - {**s, 'sample_type': 'WGS'} for s in FAMILY_2_VARIANT_SAMPLE_DATA['SNV_INDEL'] if s['sample_id'] == 'HG00731'] -) - -MULTI_PROJECT_SAMPLE_TYPES_SAMPLE_DATA = deepcopy(FAMILY_2_VARIANT_SAMPLE_DATA) -MULTI_PROJECT_SAMPLE_TYPES_SAMPLE_DATA['SNV_INDEL'].append(FAMILY_11_SAMPLE_WGS) -MULTI_PROJECT_SAMPLE_TYPES_SAMPLE_DATA['SNV_INDEL'].append({**FAMILY_11_SAMPLE_WGS, 'sample_type': 'WES'}) - -MULTI_PROJECT_MISSING_SAMPLE_DATA = deepcopy(FAMILY_2_MISSING_SAMPLE_DATA) -MULTI_PROJECT_MISSING_SAMPLE_DATA['SNV_INDEL'].append(FAMILY_11_SAMPLE_WGS) - -SV_WGS_SAMPLE_DATA_WITH_SEX = {'SV_WGS': [{'is_male': True, **FAMILY_11_SAMPLE_WGS}, { - 'sample_id': 'NA20884', 'individual_guid': 'I000025_na20884', 'family_guid': 'F000011_11', 'project_guid': 'R0003_test', 'affected': 'N', 'sample_type': 'WGS', 'is_male': True, -}, { - 'sample_id': 'NA20883', 'individual_guid': 'I000035_na20883', 'family_guid': 'F000011_11', 'project_guid': 'R0003_test', 'affected': 'N', 'sample_type': 'WGS', 'is_male': False, -}]} -SV_WGS_SAMPLE_DATA = deepcopy(SV_WGS_SAMPLE_DATA_WITH_SEX) -for s in SV_WGS_SAMPLE_DATA['SV_WGS']: - s.pop('is_male') - -SV_WES_SAMPLE_DATA = {'SV_WES': EXPECTED_SAMPLE_DATA['SV_WES'] + [FAMILY_3_SAMPLE]} - -VARIANT1 = { - 'variantId': '1-10439-AC-A', - 'chrom': '1', - 'pos': 10439, - 'ref': 'AC', - 'alt': 'A', - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '1', - 'liftedOverPos': 10439, - 'xpos': 1000010439, - 'rsid': 'rs112766696', - 'familyGuids': ['F000002_2'], - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'dp': 10, 'gq': 99, 'ab': 0.5, 'filters': [], - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'dp': 24, 'gq': 0, 'ab': 0.0, 'filters': [], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'dp': 60, 'gq': 20, 'ab': 0.0, 'filters': [], - }, - }, - 'clinvar': { - 'alleleId': 19473, - 'conflictingPathogenicities': None, - 'goldStars': None, - 'pathogenicity': 'Likely_pathogenic', - 'assertions': None, - 'submitters': None, - 'conditions': None, - 'version': '2024-02-21', - }, - 'hgmd': None, - 'screenRegionType': None, - 'populations': { - 'seqr': {'ac': 9, 'hom': 2}, - 'topmed': {'af': 0.0784199982881546, 'ac': 20757, 'an': 264690, 'hom': 0, 'het': 20757}, - 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, - 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, - 'gnomad_genomes': {'af': 0.034449315071105957, 'ac': 927, 'an': 26912, 'hom': 48, 'hemi': 0, 'filter_af': 0.040276646614074707}, - }, - 'predictions': { - 'cadd': 4.668000221252441, - 'eigen': None, - 'fathmm': None, - 'gnomad_noncoding': None, - 'mpc': None, - 'mut_pred': None, - 'primate_ai': None, - 'splice_ai': None, - 'splice_ai_consequence': None, - 'vest': None, - 'mut_taster': None, - 'polyphen': None, - 'revel': None, - 'sift': None, - }, - 'transcripts': {}, - 'sortedMotifFeatureConsequences': None, - 'sortedRegulatoryFeatureConsequences': None, - 'mainTranscriptId': None, - 'selectedMainTranscriptId': None, - '_sort': [1000010439], - 'CAID': 'CA16717152', -} - -VARIANT1_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY = deepcopy(VARIANT1) -genotypes = VARIANT1_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] -VARIANT1_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] = { - 'I000004_hg00731': [ - genotypes['I000004_hg00731'], - {**genotypes['I000004_hg00731'], 'numAlt': 2, 'sampleType': 'WGS'} - ], - 'I000005_hg00732': [genotypes['I000005_hg00732']], - 'I000006_hg00733': [genotypes['I000006_hg00733']], -} - -VARIANT1_BOTH_SAMPLE_TYPES = deepcopy(VARIANT1_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY) -genotypes = VARIANT1_BOTH_SAMPLE_TYPES['genotypes'] -VARIANT1_BOTH_SAMPLE_TYPES['genotypes']['I000005_hg00732'] = [ - *genotypes['I000005_hg00732'], - {**genotypes['I000005_hg00732'][0], 'gq': 99, 'numAlt': 1, 'sampleType': 'WGS'} -] -VARIANT1_BOTH_SAMPLE_TYPES['genotypes']['I000006_hg00733'] = [ - *genotypes['I000006_hg00733'], - {**genotypes['I000006_hg00733'][0], 'sampleType': 'WGS'} -] - -VARIANT2 = { - 'variantId': '1-38724419-T-G', - 'chrom': '1', - 'pos': 38724419, - 'ref': 'T', - 'alt': 'G', - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '1', - 'liftedOverPos': 11854476, - 'xpos': 1038724419, - 'rsid': 'rs1801131', - 'familyGuids': ['F000002_2'], - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 2, 'dp': 36, 'gq': 99, 'ab': 1.0, 'filters': [], - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'dp': 32, 'gq': 99, 'ab': 0.625, 'filters': [], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'dp': 33, 'gq': 40, 'ab': 0.0, 'filters': [], - }, - }, - 'clinvar': { - 'alleleId': 18560, - 'conflictingPathogenicities': [ - {'count': 1, 'pathogenicity': 'Likely_pathogenic'}, - {'count': 1, 'pathogenicity': 'Uncertain_significance'}, - {'count': 1, 'pathogenicity': 'Likely_benign'}, - {'count': 5, 'pathogenicity': 'Benign'}, - ], - 'goldStars': 1, - 'pathogenicity': 'Conflicting_classifications_of_pathogenicity', - 'assertions': ['other'], - 'version': '2024-02-21', - 'submitters': [ - 'Broad Center for Mendelian Genomics, Broad Institute of MIT and Harvard', - 'Illumina Laboratory Services, Illumina', - 'Blueprint Genetics', - 'GenomeConnect, ClinGen' - ], - 'conditions': [ - 'ABCA4-Related Disorders', - 'Severe early-childhood-onset retinal dystrophy', - 'not specified', - 'not provided' - ], - }, - 'hgmd': {'accession': 'CM981315', 'class': 'DFP'}, - 'screenRegionType': None, - 'populations': { - 'seqr': {'ac': 28, 'hom': 4}, - 'topmed': {'af': 0.24615199863910675, 'ac': 65154, 'an': 264690, 'hom': 8775, 'het': 47604}, - 'exac': {'af': 0.29499998688697815, 'ac': 35805, 'an': 121372, 'hom': 5872, 'hemi': 0, 'het': 24061, 'filter_af': 0.4153035283088684}, - 'gnomad_exomes': {'af': 0.28899794816970825, 'ac': 72672, 'an': 251462, 'hom': 11567, 'hemi': 0, 'filter_af': 0.4116474986076355}, - 'gnomad_genomes': {'af': 0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0}, - }, - 'predictions': { - 'cadd': 20.899999618530273, - 'eigen': 2.000999927520752, - 'fathmm': 0, - 'gnomad_noncoding': 5.868505001068115, - 'mpc': 0.28205373883247375, - 'mut_pred': None, - 'primate_ai': 0.4655807614326477, - 'splice_ai': 0.0, - 'splice_ai_consequence': 'No consequence', - 'vest': 0.210999995470047, - 'mut_taster': 'P', - 'polyphen': 0.1, - 'revel': 0.19699999690055847, - 'sift': 0.05, - }, - 'transcripts': { - 'ENSG00000177000': [ - {'aminoAcids': 'L/F', 'canonical': 1, 'codons': 'ttA/ttC', 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000383791.8:c.156A>C', 'hgvsp': 'ENSP00000373301.3:p.Leu52Phe', 'transcriptId': 'ENST00000383791', 'maneSelect': 'NM_004844.5', 'manePlusClinical': None, 'exon': {'index': 2, 'total': 9}, 'intron': None, 'alphamissense': {'pathogenicity': 0.9977999925613403}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_004844.5', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 0}, - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000408919.7:c.-384A>C', 'hgvsp': None, 'transcriptId': 'ENST00000408919', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 9}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'refseqTranscriptId': 'NM_001018009.4', 'biotype': 'protein_coding', 'majorConsequence': '5_prime_UTR_variant', 'consequenceTerms': ['5_prime_UTR_variant'], 'transcriptRank': 1, 'utrannotator': { - 'existingInframeOorfs': 0, 'existingOutofframeOorfs': 1, 'existingUorfs': 10, 'fiveutrConsequence': '5_prime_UTR_stop_codon_loss_variant', - 'fiveutrAnnotation': {'type': None, 'KozakContext': 'GCGATGC', 'KozakStrength': 'Moderate', 'DistanceToCDS': None, 'CapDistanceToStart': None, 'DistanceToStop': None, 'Evidence': False, 'AltStop': 'True', 'AltStopDistanceToCDS': 310, 'FrameWithCDS': 'outOfFrame', 'StartDistanceToCDS': None, 'newSTOPDistanceToCDS': None, 'alt_type': None, 'alt_type_length': None,'ref_StartDistanceToCDS': None, 'ref_type': None, 'ref_type_length': None}, - }}, - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000426925.5:c.-677A>C', 'hgvsp': None, 'transcriptId': 'ENST00000426925', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 11}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'refseqTranscriptId': None, 'biotype': 'protein_coding', 'majorConsequence': '5_prime_UTR_variant', 'consequenceTerms': ['5_prime_UTR_variant'], 'transcriptRank': 2, 'utrannotator': { - 'existingInframeOorfs': 0, 'existingOutofframeOorfs': 1, 'existingUorfs': 8,'fiveutrConsequence': '5_prime_UTR_stop_codon_loss_variant', - 'fiveutrAnnotation': {'type': None, 'KozakContext': 'TCAATGC', 'KozakStrength': 'Weak', 'DistanceToCDS': None, 'CapDistanceToStart': None, 'DistanceToStop': None, 'Evidence': False, 'AltStop': 'True', 'AltStopDistanceToCDS': 588, 'FrameWithCDS': 'inFrame', 'StartDistanceToCDS': None, 'newSTOPDistanceToCDS': None, 'alt_type': None, 'alt_type_length': None, 'ref_StartDistanceToCDS': None, 'ref_type': None, 'ref_type_length': None}, - }}, - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000412806.1:c.138+1722A>C', 'hgvsp': None, 'transcriptId': 'ENST00000412806', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 3},'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'nonsense_mediated_decay', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 3}, - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000459627.1:n.298A>C', 'hgvsp': None, 'transcriptId': 'ENST00000459627', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 3}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding_CDS_not_defined', 'majorConsequence': 'non_coding_transcript_exon_variant', 'consequenceTerms': ['non_coding_transcript_exon_variant'], 'transcriptRank': 4}, - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000465894.6:n.33A>C', 'hgvsp': None, 'transcriptId': 'ENST00000465894', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 5}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding_CDS_not_defined', 'majorConsequence': 'non_coding_transcript_exon_variant', 'consequenceTerms': ['non_coding_transcript_exon_variant'], 'transcriptRank': 5}, - ], - 'ENSG00000277258': [ - {'aminoAcids': 'L/F', 'canonical': None, 'codons': 'ttA/ttC', 'geneId': 'ENSG00000277258', 'hgvsc': 'ENST00000450625.1:c.156A>C', 'hgvsp': 'ENSP00000389484.1:p.Leu52Phe', 'transcriptId': 'ENST00000450625', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 5}, 'intron': None, 'alphamissense': {'pathogenicity': 0.9977999925613403}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'nonsense_mediated_decay', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant', 'NMD_transcript_variant'], 'transcriptRank': 0}, - ] - }, - 'mainTranscriptId': 'ENST00000383791', - 'sortedMotifFeatureConsequences': None, - 'sortedRegulatoryFeatureConsequences': None, - 'selectedMainTranscriptId': None, - '_sort': [1038724419], - 'CAID': None, -} - -VARIANT2_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY = deepcopy(VARIANT2) -genotypes = VARIANT2_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] -VARIANT2_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] = { - 'I000004_hg00731': [ - genotypes['I000004_hg00731'], - {**genotypes['I000004_hg00731'], 'sampleType': 'WGS'} - ], - 'I000005_hg00732': [genotypes['I000005_hg00732']], - 'I000006_hg00733': [genotypes['I000006_hg00733']], -} - -VARIANT2_BOTH_SAMPLE_TYPES = deepcopy(VARIANT2_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY) -genotypes = VARIANT2_BOTH_SAMPLE_TYPES['genotypes'] -VARIANT2_BOTH_SAMPLE_TYPES['genotypes']['I000005_hg00732'] = [ - *genotypes['I000005_hg00732'], - {**genotypes['I000005_hg00732'][0], 'numAlt': 0, 'sampleType': 'WGS'} -] -VARIANT2_BOTH_SAMPLE_TYPES['genotypes']['I000006_hg00733'] = [ - *genotypes['I000006_hg00733'], - {**genotypes['I000006_hg00733'][0], 'sampleType': 'WGS'} -] - -VARIANT3 = { - 'variantId': '1-91502721-G-A', - 'chrom': '1', - 'pos': 91502721, - 'ref': 'G', - 'alt': 'A', - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '1', - 'liftedOverPos': 91968278, - 'xpos': 1091502721, - 'rsid': 'rs13447464', - 'familyGuids': ['F000002_2'], - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'dp': 40, 'gq': 99, 'ab': 1.0, 'filters': [], - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'dp': 37, 'gq': 99, 'ab': 0.4594594594594595, 'filters': [], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'dp': 27, 'gq': 99, 'ab': 0.4074074074074074, 'filters': [], - }, - }, - 'clinvar': None, - 'hgmd': None, - 'screenRegionType': None, - 'populations': { - 'seqr': {'ac': 4, 'hom': 1}, - 'topmed': {'af': 0.36268100142478943, 'ac': 95998, 'an': 264690, 'hom': 19369, 'het': 57260}, - 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, - 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, - 'gnomad_genomes': {'af': 0.38041073083877563, 'ac': 57757, 'an': 151828, 'hom': 12204, 'hemi': 0, 'filter_af': 0.4797786474227905}, - }, - 'predictions': { - 'cadd': 2.753999948501587, - 'eigen': 1.378000020980835, - 'fathmm': None, - 'gnomad_noncoding': 0.7389647960662842, - 'mpc': None, - 'mut_pred': None, - 'primate_ai': None, - 'splice_ai': 0.009999999776482582, - 'splice_ai_consequence': 'Donor gain', - 'vest': None, - 'mut_taster': None, - 'polyphen': None, - 'revel': None, - 'sift': None, - }, - 'transcripts': { - 'ENSG00000097046': [ - {'aminoAcids': None, 'canonical': 1, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000234626.11:c.-63-251G>A', 'hgvsp': None, 'transcriptId': 'ENST00000234626', 'maneSelect': 'NM_003503.4', 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 11}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_003503.4', 'biotype': 'protein_coding', 'majorConsequence': 'intron_variant', 'consequenceTerms': ['intron_variant'], 'transcriptRank': 0}, - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000428239.5:c.-64+100G>A', 'hgvsp': None, 'transcriptId': 'ENST00000428239', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 11}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_001134420.2', 'biotype': 'protein_coding', 'majorConsequence': 'intron_variant', 'consequenceTerms': ['intron_variant'], 'transcriptRank': 1}, - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000497611.1:n.244G>A', 'hgvsp': None, 'transcriptId': 'ENST00000497611', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 1, 'total': 4}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding_CDS_not_defined', 'majorConsequence': 'non_coding_transcript_exon_variant', 'consequenceTerms': ['non_coding_transcript_exon_variant'], 'transcriptRank': 2}, - ], - 'ENSG00000177000': [ - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000426137.1:c.-64+100G>A', 'hgvsp': None, 'transcriptId': 'ENST00000426137', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 5}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding', 'majorConsequence': 'intron_variant', 'consequenceTerms': ['intron_variant'], 'transcriptRank': 0}, - ], - }, - 'mainTranscriptId': 'ENST00000234626', - 'sortedMotifFeatureConsequences': None, - 'sortedRegulatoryFeatureConsequences': [{'biotype': 'promoter', 'consequenceTerms': ['regulatory_region_variant'], 'regulatoryFeatureId': 'ENSR00000009706'}], - 'selectedMainTranscriptId': None, - '_sort': [1091502721], - 'CAID': 'CA10960369', -} - -VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY = deepcopy(VARIANT3) -genotypes = VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] -VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] = { - 'I000004_hg00731': [ - genotypes['I000004_hg00731'], - {**genotypes['I000004_hg00731'], 'sampleType': 'WGS'} - ], - 'I000005_hg00732': [genotypes['I000005_hg00732']], - 'I000006_hg00733': [genotypes['I000006_hg00733']], -} - -VARIANT3_BOTH_SAMPLE_TYPES = deepcopy(VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY) -genotypes = VARIANT3_BOTH_SAMPLE_TYPES['genotypes'] -VARIANT3_BOTH_SAMPLE_TYPES['genotypes']['I000005_hg00732'] = [ - *genotypes['I000005_hg00732'], - {**genotypes['I000005_hg00732'][0], 'sampleType': 'WGS'} -] -VARIANT3_BOTH_SAMPLE_TYPES['genotypes']['I000006_hg00733'] = [ - *genotypes['I000006_hg00733'], - {**genotypes['I000006_hg00733'][0], 'sampleType': 'WGS'} -] - -VARIANT4 = { - 'variantId': '1-91511686-T-G', - 'chrom': '1', - 'pos': 91511686, - 'ref': 'T', - 'alt': 'G', - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '1', - 'liftedOverPos': 91977243, - 'xpos': 1091511686, - 'rsid': None, - 'familyGuids': ['F000002_2'], - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'dp': 29, 'gq': 58, 'ab': 0.1724137931034483, 'filters': ['VQSRTrancheSNP99.95to100.00'], - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'dp': 24, 'gq': 0, 'ab': 0.0, 'filters': ['VQSRTrancheSNP99.95to100.00'], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'dp': 45, 'gq': 0, 'ab': 0.0, 'filters': ['VQSRTrancheSNP99.95to100.00'], - }, - }, - 'clinvar': None, - 'hgmd': None, - 'screenRegionType': 'CTCF-only', - 'populations': { - 'seqr': {'ac': 2, 'hom': 0}, - 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, - 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, - 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, - 'gnomad_genomes': {'af': 0.00026519427774474025, 'ac': 39, 'an': 147062, 'hom': 0, 'hemi': 0, 'filter_af': 0.0015030059730634093}, - }, - 'predictions': { - 'cadd': 29.899999618530273, - 'eigen': 9.491000175476074, - 'fathmm': 0, - 'gnomad_noncoding': 0.2300506979227066, - 'mpc': 0.8326827883720398, - 'mut_pred': 0.6869999766349792, - 'primate_ai': 0.6995947360992432, - 'splice_ai': 0.0, - 'splice_ai_consequence': 'No consequence', - 'vest': 0.8579999804496765, - 'mut_taster': 'N', - 'polyphen': 0, - 'revel': 0.5260000228881836, - 'sift': 0, - }, - 'transcripts': { - 'ENSG00000097046': [ - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000466716.5:c.-264+1G>A', 'hgvsp': None, 'transcriptId': 'ENST00000466716', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 3}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': ['5UTR_SPLICE']}, 'spliceregion': {'extended_intronic_splice_region_variant': True}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding', 'majorConsequence': 'splice_donor_variant', 'consequenceTerms': ['splice_donor_variant'], 'transcriptRank': 0}, - {'aminoAcids': None, 'canonical': 1, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000350997.12:c.375+139G>A', 'hgvsp': None, 'transcriptId': 'ENST00000350997', 'maneSelect': 'NM_013402.7', 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 11}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_013402.7', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 1}, - {'aminoAcids': 'T/I', 'canonical': None, 'codons': 'aCc/aTc', 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000257261.10:c.131C>T', 'hgvsp': 'ENSP00000257261.6:p.Thr44Ile', 'transcriptId': 'ENST00000257261', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 1, 'total': 12}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_001281501.1', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 2}, - ], - }, - 'mainTranscriptId': 'ENST00000466716', - 'sortedMotifFeatureConsequences': [ - {'consequenceTerms': ['TF_binding_site_variant'], 'motifFeatureId': 'ENSM00093424674'}, - {'consequenceTerms': ['TF_binding_site_variant'], 'motifFeatureId': 'ENSM00036268032'}, - ], - 'sortedRegulatoryFeatureConsequences': [ - {'biotype': 'promoter', 'consequenceTerms': ['regulatory_region_variant'], 'regulatoryFeatureId': 'ENSR00000040341'}, - ], - 'selectedMainTranscriptId': None, - '_sort': [1091511686], - 'CAID': 'CA341062623', -} - -VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY = deepcopy(VARIANT4) -genotypes = VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] -VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] = { - 'I000004_hg00731': [ - genotypes['I000004_hg00731'], - {**genotypes['I000004_hg00731'], 'sampleType': 'WGS'} - ], - 'I000005_hg00732': [genotypes['I000005_hg00732']], - 'I000006_hg00733': [genotypes['I000006_hg00733']], -} - -VARIANT4_BOTH_SAMPLE_TYPES = deepcopy(VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY) -genotypes = VARIANT4_BOTH_SAMPLE_TYPES['genotypes'] -VARIANT4_BOTH_SAMPLE_TYPES['genotypes']['I000005_hg00732'] = [ - *genotypes['I000005_hg00732'], - {**genotypes['I000005_hg00732'][0], 'sampleType': 'WGS'} -] -VARIANT4_BOTH_SAMPLE_TYPES['genotypes']['I000006_hg00733'] = [ - *genotypes['I000006_hg00733'], - {**genotypes['I000006_hg00733'][0], 'numAlt': 2, 'sampleType': 'WGS'} -] - -VARIANT_LOOKUP_VARIANT = { - **VARIANT1, - 'liftedFamilyGuids': ['F000014_14'], - 'familyGenotypes': { - VARIANT1['familyGuids'][0]: sorted([ - {k: v for k, v in g.items() if k != 'individualGuid'} for g in VARIANT1['genotypes'].values() - ], key=lambda x: x['sampleId'], reverse=True), - 'F000011_11': [{ - 'sampleId': 'NA20885', 'sampleType': 'WES', 'familyGuid': 'F000011_11', - 'numAlt': 2, 'dp': 6, 'gq': 16, 'ab': 1.0, 'filters': [], - }], - 'F000014_14': [{ - 'sampleId': 'NA21234', 'sampleType': 'WGS', 'familyGuid': 'F000014_14', - 'numAlt': 1, 'dp': 27, 'gq': 87, 'ab': 0.531000018119812, 'filters': None, - }], - } -} -for k in {'familyGuids', 'genotypes'}: - VARIANT_LOOKUP_VARIANT.pop(k) - -FAMILY_3_VARIANT = deepcopy(VARIANT3) -FAMILY_3_VARIANT['familyGuids'] = ['F000003_3'] -FAMILY_3_VARIANT['genotypes'] = { - 'I000007_na20870': { - 'sampleId': 'NA20870', 'sampleType': 'WES', 'individualGuid': 'I000007_na20870', 'familyGuid': 'F000003_3', - 'numAlt': 1, 'dp': 28, 'gq': 99, 'ab': 0.6785714285714286, 'filters': [], - }, -} -MULTI_FAMILY_VARIANT = deepcopy(VARIANT3) -MULTI_FAMILY_VARIANT['familyGuids'] += FAMILY_3_VARIANT['familyGuids'] -MULTI_FAMILY_VARIANT['genotypes'].update(FAMILY_3_VARIANT['genotypes']) - -HAIL_BACKEND_VARIANTS = [VARIANT2, MULTI_FAMILY_VARIANT] -HAIL_BACKEND_SINGLE_FAMILY_VARIANTS = [VARIANT2, VARIANT3] - -PROJECT_2_VARIANT = { - 'variantId': '1-10146-ACC-A', - 'chrom': '1', - 'pos': 10146, - 'ref': 'ACC', - 'alt': 'A', - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '1', - 'liftedOverPos': 10146, - 'xpos': 1000010146, - 'rsid': 'rs375931351', - 'familyGuids': ['F000011_11'], - 'genotypes': { - 'I000015_na20885': { - 'sampleId': 'NA20885', 'sampleType': 'WES', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'dp': 8, 'gq': 14, 'ab': 0.875, 'filters': [], - } - }, - 'clinvar': None, - 'hgmd': None, - 'screenRegionType': None, - 'populations': { - 'seqr': {'ac': 0, 'hom': 0}, - 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, - 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, - 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, - 'gnomad_genomes': {'af': 0.00012430080096237361, 'ac': 2, 'an': 16090, 'hom': 0, 'hemi': 0, 'filter_af': 0.002336448524147272}, - }, - 'predictions': { - 'cadd': 4.6529998779296875, - 'eigen': None, - 'fathmm': None, - 'gnomad_noncoding': None, - 'mpc': None, - 'mut_pred': None, - 'primate_ai': None, - 'splice_ai': None, - 'splice_ai_consequence': None, - 'vest': None, - 'mut_taster': None, - 'polyphen': None, - 'revel': None, - 'sift': None, - }, - 'transcripts': {}, - 'mainTranscriptId': None, - 'selectedMainTranscriptId': None, - 'sortedMotifFeatureConsequences': None, - 'sortedRegulatoryFeatureConsequences': None, - '_sort': [1000010146], - 'CAID': 'CA520798130', -} - -GRCH37_VARIANT = { - 'variantId': '7-143270172-A-G', - 'xpos': 7143270172, - 'chrom': '7', - 'pos': 143270172, - 'ref': 'A', - 'alt': 'G', - 'genomeVersion': '37', - 'rsid': 'rs72611576', - 'familyGuids': ['F000002_2'], - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WGS', 'individualGuid': 'I000004_hg00731', - 'familyGuid': 'F000002_2', 'numAlt': 2, 'dp': 16, 'gq': 48, 'ab': 1, 'filters': ['VQSRTrancheSNP99.90to99.95'], - }, 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000006_hg00733', - 'familyGuid': 'F000002_2', 'numAlt': 1, 'dp': 49, 'gq': 99, 'ab': 0.6530612111091614, - 'filters': ['VQSRTrancheSNP99.90to99.95'], - }, - }, - 'populations': { - 'seqr': {'ac': 4711, 'hom': 1508}, - 'topmed': {'af': 0.5213189721107483, 'ac': 65461, 'an': 125568, 'hom': 16156, 'het': 33149}, - 'exac': {'af': 0.6299999952316284, 'ac': 66593, 'an': 104352, 'hom': 22162, 'hemi': 0, 'het': 22269, 'filter_af': 0.8198773860931396}, - 'gnomad_exomes': {'af': 0.6354219317436218, 'ac': 137532, 'an': 216442, 'hom': 45869, 'hemi': 0, 'filter_af': 0.8226116299629211}, - 'gnomad_genomes': {'af': 0.6136477589607239, 'ac': 14649, 'an': 23872, 'hom': 4584, 'hemi': 0, 'filter_af': 0.828438937664032}, - }, - 'predictions': { - 'cadd': 13.020000457763672, 'eigen': 3.9509999752044678, 'primate_ai': 0.4906357526779175, - 'splice_ai': 0, 'splice_ai_consequence': 'No consequence', - 'mpc': None, 'mut_taster': None, 'polyphen': None, 'revel': None, 'sift': None, - }, - 'clinvar': None, - 'hgmd': None, - 'transcripts': { - 'ENSG00000271079': [ - {'aminoAcids': 'E/G', 'canonical': 1, 'codons': 'gAa/gGa', 'geneId': 'ENSG00000271079', - 'hgvsc': 'ENST00000420911.2:c.1262A>G', 'hgvsp': 'ENSP00000474204.1:p.Glu421Gly', - 'transcriptId': 'ENST00000420911', 'isLofNagnag': None, 'transcriptRank': 0, 'consequenceTerms': ['missense_variant'], - 'biotype': 'protein_coding', 'lofFilters': None, 'majorConsequence': 'missense_variant'}, - ], - 'ENSG00000176227': [ - {'aminoAcids': None, 'canonical': 1, 'codons': None, 'geneId': 'ENSG00000176227', - 'hgvsc': 'ENST00000447022.1:n.1354A>G', 'hgvsp': None, - 'transcriptId': 'ENST00000447022', 'isLofNagnag': None, 'transcriptRank': 0, - 'biotype': 'processed_pseudogene', 'lofFilters': None, 'majorConsequence': 'non_coding_transcript_exon_variant', - 'consequenceTerms': ['non_coding_transcript_exon_variant', 'non_coding_transcript_variant']}, - ], - }, - 'mainTranscriptId': 'ENST00000420911', - 'selectedMainTranscriptId': None, - '_sort': [7143270172], - 'CAID': 'CA4540310', -} - -SV_VARIANT1 = { - 'variantId': 'phase2_DEL_chr1_625', - 'chrom': '1', - 'endChrom': None, - 'pos': 9310023, - 'end': 9310127, - 'rg37LocusEnd': {'contig': '1', 'position': 9370186}, - 'xpos': 1009310023, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '1', - 'liftedOverPos': 9370082, - 'algorithms': 'manta', - 'bothsidesSupport': False, - 'familyGuids': ['F000011_11'], - 'genotypes': { - 'I000015_na20885': { - 'sampleId': 'NA20885', 'sampleType': 'WGS', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'cn': 2, 'gq': 0, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, 'I000025_na20884': { - 'sampleId': 'NA20884', 'sampleType': 'WGS', 'individualGuid': 'I000025_na20884', 'familyGuid': 'F000011_11', - 'numAlt': 0, 'cn': 2, 'gq': 6, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, 'I000035_na20883': { - 'sampleId': 'NA20883', 'sampleType': 'WGS', 'individualGuid': 'I000035_na20883', 'familyGuid': 'F000011_11', - 'numAlt': 0, 'cn': 2, 'gq': 99, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, - }, - 'populations': { - 'sv_callset': {'af': 0.02421800047159195, 'ac': 141, 'an': 5822, 'hom': 0, 'het': 141}, - 'gnomad_svs': {'af': 0.0, 'id': '', 'hom': 0, 'het': 0}, - }, - 'predictions': {'strvctvre': None}, - 'cpxIntervals': None, - 'svSourceDetail': None, - 'svType': 'DEL', - 'svTypeDetail': None, - 'transcripts': { - 'ENSG00000171621': [{'geneId': 'ENSG00000171621', 'majorConsequence': 'INTRONIC'}], - }, - '_sort': [1009310023], -} -SV_VARIANT2 = { - 'variantId': 'cohort_2911.chr1.final_cleanup_INS_chr1_160', - 'chrom': '1', - 'endChrom': None, - 'pos': 9380254, - 'end': 9380286, - 'rg37LocusEnd': {'contig': '4', 'position': 9382012}, - 'xpos': 1009380254, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '1', - 'liftedOverPos': 9440313, - 'algorithms': 'manta', - 'bothsidesSupport': True, - 'familyGuids': ['F000011_11'], - 'genotypes': { - 'I000015_na20885': { - 'sampleId': 'NA20885', 'sampleType': 'WGS', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'cn': None, 'gq': 0, 'newCall': True, 'prevCall': False, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, 'I000025_na20884': { - 'sampleId': 'NA20884', 'sampleType': 'WGS', 'individualGuid': 'I000025_na20884', 'familyGuid': 'F000011_11', - 'numAlt': 0, 'cn': None, 'gq': 99, 'newCall': True, 'prevCall': False, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, 'I000035_na20883': { - 'sampleId': 'NA20883', 'sampleType': 'WGS', 'individualGuid': 'I000035_na20883', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'cn': None, 'gq': 0, 'newCall': True, 'prevCall': False, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, - }, - 'populations': { - 'sv_callset': {'af': 0.199072003364563, 'ac': 1159, 'an': 5822, 'hom': 0, 'het': 1159}, - 'gnomad_svs': {'af': 0.005423000082373619, 'id': 'gnomAD-SV_v3_INS_1_299', 'hom': 10359, 'het': 35634}, - }, - 'predictions': {'strvctvre': None}, - 'cpxIntervals': None, - 'svSourceDetail': {'chrom': '4'}, - 'svType': 'INS', - 'svTypeDetail': None, - 'transcripts': { - 'ENSG00000171621': [{'geneId': 'ENSG00000171621', 'majorConsequence': 'NEAREST_TSS'}], - }, - '_sort': [1009380254], -} -SV_VARIANT3 = { - 'variantId': 'phase2_CPX_chr13_35', - 'chrom': '13', - 'endChrom': '17', - 'pos': 63036002, - 'end': 63036041, - 'rg37LocusEnd': {'contig': '17', 'position': 61113402}, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '13', - 'liftedOverPos': 63610135, - 'xpos': 13063036002, - 'algorithms': 'manta', - 'bothsidesSupport': False, - 'familyGuids': ['F000011_11'], - 'genotypes': { - 'I000015_na20885': { - 'sampleId': 'NA20885', 'sampleType': 'WGS', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'cn': None, 'gq': 62, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, 'I000025_na20884': { - 'sampleId': 'NA20884', 'sampleType': 'WGS', 'individualGuid': 'I000025_na20884', 'familyGuid': 'F000011_11', - 'numAlt': 2, 'cn': None, 'gq': 42, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, 'I000035_na20883': { - 'sampleId': 'NA20883', 'sampleType': 'WGS', 'individualGuid': 'I000035_na20883', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'cn': None, 'gq': 79, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, - }, 'populations': { - 'sv_callset': {'af': 0.5778080224990845, 'ac': 3364, 'an': 5822, 'hom': 459, 'het': 2446}, - 'gnomad_svs': {'af': 0.0, 'id': '', 'hom': 0, 'het': 0}, - }, - 'predictions': {'strvctvre': None}, - 'cpxIntervals': [{'chrom': 'chr17', 'start': 22150735, 'end': 22151179, 'type': 'DUP'}], - 'svSourceDetail': None, - 'svType': 'CPX', - 'svTypeDetail': 'dDUP', - 'transcripts': { - 'ENSG00000083544': [{'geneId': 'ENSG00000083544', 'majorConsequence': 'NEAREST_TSS'}], - 'null': [{'geneId': None, 'majorConsequence': 'NEAREST_TSS'}], - }, - '_sort': [13063036002], -} -SV_VARIANT4 = { - 'variantId': 'phase2_DEL_chr14_4640', - 'chrom': '14', - 'endChrom': None, - 'pos': 106694244, - 'end': 106740587, - 'rg37LocusEnd': None, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '14', - 'liftedOverPos': 107150261, - 'xpos': 14106694244, - 'algorithms': 'depth,manta', - 'bothsidesSupport': False, - 'familyGuids': ['F000011_11'], - 'genotypes': { - 'I000015_na20885': { - 'sampleId': 'NA20885', 'sampleType': 'WGS', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 2, 'cn': 0, 'gq': 99, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': [], - }, 'I000025_na20884': { - 'sampleId': 'NA20884', 'sampleType': 'WGS', 'individualGuid': 'I000025_na20884', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'cn': 1, 'gq': 99, 'newCall': False, 'prevCall': False, 'prevNumAlt': 2, 'filters': [], - }, 'I000035_na20883': { - 'sampleId': 'NA20883', 'sampleType': 'WGS', 'individualGuid': 'I000035_na20883', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'cn': 1, 'gq': 99, 'newCall': False, 'prevCall': False, 'prevNumAlt': 2, 'filters': [], - }, - }, - 'populations': { - 'sv_callset': {'af': 0.6100999712944031, 'ac': 3552, 'an': 5822, 'hom': 1053, 'het': 1446}, - 'gnomad_svs': {'af': 0.0, 'id': '', 'hom': 0, 'het': 0}, - }, - 'predictions': {'strvctvre': 0.16099999845027924}, - 'cpxIntervals': None, - 'svSourceDetail': None, - 'svType': 'DEL', - 'svTypeDetail': None, - 'transcripts': { - 'ENSG00000184986': [{'geneId': 'ENSG00000184986', 'majorConsequence': 'NEAREST_TSS'}], - }, - '_sort': [14106694244], -} -GCNV_VARIANT1 = { - 'variantId': 'suffix_95340_DUP', - 'chrom': '14', - 'pos': 22438910, - 'end': 22469796, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '14', - 'liftedOverPos': 22886546, - 'rg37LocusEnd': {'contig': '14', 'position': 23058228}, - 'xpos': 14022417556, - 'familyGuids': ['F000002_2'], - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'cn': 3, 'qs': 38, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'cn': None, 'qs': None, 'defragged': None, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': None, 'prevCall': None, 'prevOverlap': None, 'filters': [], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'cn': None, 'qs': None, 'defragged': None, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': None, 'prevCall': None, 'prevOverlap': None, 'filters': [], - } - }, - 'populations': {'sv_callset': {'af': 0.076492540538311, 'ac': 1763, 'an': 23048, 'hom': 0, 'het': 0}}, - 'predictions': {'strvctvre': 0.1809999942779541}, - 'numExon': 0, - 'svType': 'DUP', - 'transcripts': {}, - '_sort': [14022417556], - -} -GCNV_VARIANT2 = { - 'variantId': 'suffix_124520_DUP', - 'chrom': '16', - 'pos': 29809156, - 'end': 29815990, - 'xpos': 16029802672, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '16', - 'liftedOverPos': 29813993, - 'rg37LocusEnd': {'contig': '16', 'position': 29831761}, - 'familyGuids': ['F000002_2'], - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'cn': 3, 'qs': 29, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'cn': 3, 'qs': 46, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'cn': 3, 'qs': 37, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], - } - }, - 'populations': {'sv_callset': {'af': 0.012322110123932362, 'ac': 284, 'an': 23047, 'hom': 0, 'het': 0}}, - 'predictions': {'strvctvre': 0.5479999780654907}, - 'numExon': 8, - 'svType': 'DUP', - 'transcripts': { - 'ENSG00000103495': [{'geneId': 'ENSG00000103495', 'majorConsequence': 'COPY_GAIN'}], - 'ENSG00000167371': [{'geneId': 'ENSG00000167371', 'majorConsequence': 'COPY_GAIN'}], - 'ENSG00000280893': [{'geneId': 'ENSG00000280893', 'majorConsequence': 'COPY_GAIN'}], - }, - '_sort': [16029802672], -} -GCNV_VARIANT3 = { - 'variantId': 'suffix_140593_DUP', - 'chrom': '17', - 'pos': 38717327, - 'end': 38719636, - 'xpos': 17038717327, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '17', - 'liftedOverPos': 36873580, - 'rg37LocusEnd': {'contig': '17', 'position': 36876246}, - 'familyGuids': ['F000002_2'], - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 2, 'cn': 4, 'qs': 13, 'defragged': True, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': True, 'prevCall': False, 'prevOverlap': False, 'filters': [], - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'cn': 3, 'qs': 7, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': False, 'prevOverlap': True, 'filters': [], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', 'numAlt': 0, - 'cn': None, 'qs': None, 'defragged': None, 'start': None, 'end': None, 'numExon': None, 'geneIds': None, - 'newCall': None, 'prevCall': None, 'prevOverlap': None, 'filters': [], - }, - }, - 'populations': {'sv_callset': {'af': 0.0015185698866844177, 'ac': 35, 'an': 23048, 'hom': 0, 'het': 0}}, - 'predictions': {'strvctvre': 0.7860000133514404}, - 'numExon': 3, - 'svType': 'DUP', - 'transcripts': { - 'ENSG00000275023': [{'geneId': 'ENSG00000275023', 'majorConsequence': 'LOF'}], - }, - '_sort': [17038717327], -} -GCNV_VARIANT4 = { - 'variantId': 'suffix_140608_DUP', - 'chrom': '17', - 'pos': 38721781, - 'end': 38735703, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '17', - 'liftedOverPos': 36878034, - 'rg37LocusEnd': {'contig': '17', 'position': 36892521}, - 'familyGuids': ['F000002_2'], - 'xpos': 17038721781, - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'cn': 3, 'qs': 28, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'cn': None, 'qs': None, 'defragged': None, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': None, 'prevCall': None, 'prevOverlap': None, 'filters': [], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'cn': 3, 'qs': 29, 'defragged': False, 'start': None, 'end': 38734440, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], - } - }, - 'populations': {'sv_callset': {'af': 0.004989586770534515, 'ac': 115, 'an': 23048, 'hom': 0, 'het': 0}}, - 'predictions': {'strvctvre': 0.7099999785423279}, - 'numExon': 7, - 'svType': 'DEL', - 'transcripts': { - 'ENSG00000275023': [{'geneId': 'ENSG00000275023', 'majorConsequence': 'LOF'}], - 'ENSG00000277258': [{'geneId': 'ENSG00000277258', 'majorConsequence': 'LOF'}], - 'ENSG00000277972': [{'geneId': 'ENSG00000277972', 'majorConsequence': 'COPY_GAIN'}], - }, - '_sort': [17038721781], -} - -GCNV_MULTI_FAMILY_VARIANT1 = deepcopy(GCNV_VARIANT1) -GCNV_MULTI_FAMILY_VARIANT1.update({ - 'pos': 22418039, - 'end': 22507821, - 'transcripts': { - 'ENSG00000129562': [{'geneId': 'ENSG00000129562', 'majorConsequence': 'COPY_GAIN'}], - }, -}) -GCNV_MULTI_FAMILY_VARIANT1['familyGuids'].append('F000003_3') -GCNV_MULTI_FAMILY_VARIANT1['genotypes'].update({'I000007_na20870': { - 'sampleId': 'NA20870', 'sampleType': 'WES', 'individualGuid': 'I000007_na20870', 'familyGuid': 'F000003_3', - 'numAlt': 1, 'cn': 3, 'qs': 164, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], -}}) -GCNV_MULTI_FAMILY_VARIANT1['genotypes']['I000004_hg00731'].update({'start': 22438910, 'end': 22469796, 'geneIds': []}) - -GCNV_MULTI_FAMILY_VARIANT2 = deepcopy(GCNV_VARIANT2) -GCNV_MULTI_FAMILY_VARIANT2['numExon'] = 26 -GCNV_MULTI_FAMILY_VARIANT2['familyGuids'].append('F000003_3') -for genotype in GCNV_MULTI_FAMILY_VARIANT2['genotypes'].values(): - genotype.update({'numExon': 8, 'geneIds': ['ENSG00000103495', 'ENSG00000167371', 'ENSG00000280893']}) -GCNV_MULTI_FAMILY_VARIANT2['genotypes'].update({'I000007_na20870': { - 'sampleId': 'NA20870', 'sampleType': 'WES', 'individualGuid': 'I000007_na20870', 'familyGuid': 'F000003_3', - 'numAlt': 1, 'cn': 3, 'qs': 40, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], -}}) -GCNV_MULTI_FAMILY_VARIANT2['transcripts'].update({ - 'ENSG00000013364': [{'geneId': 'ENSG00000013364', 'majorConsequence': 'LOF'}], - 'ENSG00000079616': [{'geneId': 'ENSG00000079616', 'majorConsequence': 'LOF'}], - 'ENSG00000281348': [{'geneId': 'ENSG00000281348', 'majorConsequence': 'LOF'}], - 'ENSG00000280789': [{'geneId': 'ENSG00000280789', 'majorConsequence': 'LOF'}], -}) - -MITO_VARIANT1 = { - 'variantId': 'M-4429-G-A', - 'chrom': 'M', - 'pos': 4429, - 'ref': 'G', - 'alt': 'A', - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': 'MT', - 'liftedOverPos': 4429, - 'xpos': 25000004429, - 'rsid': 'rs1603219456', - 'familyGuids': ['F000002_2'], - 'genotypes': {'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'dp': 3955, 'hl': 0.083, 'mitoCn': 230, 'contamination': 0.0, 'filters': [], - }}, - 'populations': { - 'seqr': {'af': 0.0, 'ac': 0, 'an': 154}, - 'seqr_heteroplasmy': {'af': 0.006493506487458944, 'ac': 1, 'an': 154}, - 'gnomad_mito': {'af': 0.0, 'ac': 0, 'an': 56419}, - 'gnomad_mito_heteroplasmy': {'af': 0.0, 'ac': 0, 'an': 56419, 'max_hl': 0.0}, - 'helix': {'af': 0.0, 'ac': 0, 'an': 195983}, - 'helix_heteroplasmy': {'af': 1.530745066702366e-05, 'ac': 3, 'an': 195983, 'max_hl': 0.20634999871253967}, - }, - 'predictions': { - 'apogee': None, - 'haplogroup_defining': None, - 'hmtvar': 0.05000000074505806, - 'mitotip': 'likely_pathogenic', - 'mut_taster': None, - 'sift': None, - 'mlc': 3.38874, - }, - 'commonLowHeteroplasmy': False, - 'mitomapPathogenic': None, - 'clinvar': { - 'alleleId': 677748, - 'conflictingPathogenicities': None, - 'goldStars': 1, - 'pathogenicity': 'Uncertain_significance', - 'assertions': [], - 'version': '2024-02-21', - }, - 'transcripts': { - 'ENSG00000210112': [ - {'aminoAcids': None, 'canonical': 1, 'codons': None, 'geneId': 'ENSG00000210112', 'hgvsc': 'ENST00000387377.1:n.28G>A', 'hgvsp': None, 'transcriptId': 'ENST00000387377', 'isLofNagnag': None, 'transcriptRank': 0, 'biotype': 'Mt_tRNA', 'lofFilters': None, 'majorConsequence': 'non_coding_transcript_exon_variant', 'consequenceTerms': ['non_coding_transcript_exon_variant']}, - ], - }, - 'mainTranscriptId': 'ENST00000387377', - 'selectedMainTranscriptId': None, - '_sort': [25000004429], -} -MITO_VARIANT2 = { - 'variantId': 'M-11597-A-G', - 'chrom': 'M', - 'pos': 11597, - 'ref': 'A', - 'alt': 'G', - 'xpos': 25000011597, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': 'MT', - 'liftedOverPos': 11597, - 'rsid': None, - 'familyGuids': ['F000002_2'], - 'genotypes': {'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'dp': 3845, 'hl': 0.029, 'mitoCn': 247, 'contamination': 0.015, 'filters': [], - }}, - 'populations': { - 'seqr': {'af': 0.0, 'ac': 0, 'an': 154}, - 'seqr_heteroplasmy': {'af': 0.006493506487458944, 'ac': 1, 'an': 154}, - 'gnomad_mito': {'af': 0.0, 'ac': 0, 'an': 0}, - 'gnomad_mito_heteroplasmy': {'af': 0.0, 'ac': 0, 'an': 0, 'max_hl': 0.0}, - 'helix': {'af': 0.0, 'ac': 0, 'an': 0}, - 'helix_heteroplasmy': {'af': 0.0, 'ac': 0, 'an': 0, 'max_hl': 0.0}, - }, - 'predictions': { - 'apogee': 0.5799999833106995, - 'haplogroup_defining': None, - 'hmtvar': 0.75, - 'mitotip': None, - 'mut_taster': 'N', - 'sift': 0, - 'mlc': None, - }, - 'commonLowHeteroplasmy': False, - 'mitomapPathogenic': None, - 'clinvar': None, - 'transcripts': { - 'ENSG00000198886': [ - {'aminoAcids': 'T/A', 'canonical': 1, 'codons': 'Aca/Gca', 'geneId': 'ENSG00000198886', 'hgvsc': 'ENST00000361381.2:c.838A>G', 'hgvsp': 'ENSP00000354961.2:p.Thr280Ala', 'transcriptId': 'ENST00000361381', 'isLofNagnag': None, 'transcriptRank': 0, 'biotype': 'protein_coding', 'lofFilters': None, 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant']}, - ], - }, - 'mainTranscriptId': 'ENST00000361381', - 'selectedMainTranscriptId': None, - '_sort': [25000011597], -} -MITO_VARIANT3 = { - 'variantId': 'M-14783-T-C', - 'chrom': 'M', - 'pos': 14783, - 'ref': 'T', - 'alt': 'C', - 'xpos': 25000014783, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': 'MT', - 'liftedOverPos': 14783, - 'rsid': 'rs193302982', - 'familyGuids': ['F000002_2'], - 'genotypes': {'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 2, 'dp': 3943, 'hl': 1.0, 'mitoCn': 214, 'contamination': 0.0, 'filters': ['artifact_prone_site'], - }}, - 'populations': { - 'seqr': {'af': 0.019480518996715546, 'ac': 3, 'an': 154}, - 'seqr_heteroplasmy': {'af': 0.006493506487458944, 'ac': 1, 'an': 154}, - 'gnomad_mito': {'af': 0.05534649267792702, 'ac': 3118, 'an': 56336}, - 'gnomad_mito_heteroplasmy': {'af': 5.3251918870955706e-05, 'ac': 3, 'an': 56336, 'max_hl': 1.0}, - 'helix': {'af': 0.04884607344865799, 'ac': 9573, 'an': 195983}, - 'helix_heteroplasmy': {'af': 9.184470400214195e-05, 'ac': 18, 'an': 195983, 'max_hl': 0.962689995765686}, - }, - 'predictions': { - 'apogee': None, - 'haplogroup_defining': 'Y', - 'hmtvar': None, - 'mitotip': None, - 'mut_taster': None, - 'sift': None, - 'mlc': 0.7514, - }, - 'commonLowHeteroplasmy': True, - 'mitomapPathogenic': True, - 'clinvar': { - 'alleleId': 150280, - 'conflictingPathogenicities': None, - 'goldStars': 0, - 'pathogenicity': 'Likely_pathogenic', - 'assertions': [], - 'version': '2024-02-21', - }, - 'transcripts': { - 'ENSG00000198727': [ - {'aminoAcids': 'L', 'canonical': 1, 'codons': 'Tta/Cta', 'geneId': 'ENSG00000198727', 'hgvsc': 'ENST00000361789.2:c.37T>C', 'hgvsp': 'ENSP00000354554.2:p.Leu13=', 'transcriptId': 'ENST00000361789', 'isLofNagnag': None, 'transcriptRank': 0, 'biotype': 'protein_coding', 'lofFilters': None, 'majorConsequence': 'synonymous_variant', 'consequenceTerms': ['synonymous_variant']}, - ], - }, - 'mainTranscriptId': 'ENST00000361789', - 'selectedMainTranscriptId': None, - '_sort': [25000014783], -} - -LOCATION_SEARCH = { - 'gene_ids': ['ENSG00000097046', 'ENSG00000177000'], - 'intervals': [['2', 1234, 5678], ['7', 1, 11100], ['1', 11785723, 11806455], ['1', 91500851, 91525764]], -} -EXCLUDE_LOCATION_SEARCH = {'intervals': LOCATION_SEARCH['intervals'], 'exclude_intervals': True} -VARIANT_ID_SEARCH = {'variant_ids': [['1', 10439, 'AC', 'A'], ['1', 91511686, 'TCA', 'G']], 'rs_ids': []} -RSID_SEARCH = {'variant_ids': [], 'rs_ids': ['rs1801131']} - -GENE_COUNTS = { - 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 3, 'families': {'F000002_2': 2, 'F000011_11': 1}}, - 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 1, 'F000011_11': 1}}, -} - - -def get_hail_search_body(genome_version='GRCh38', num_results=100, sample_data=None, omit_data_type=None, **search_body): - sample_data = sample_data or EXPECTED_SAMPLE_DATA - if omit_data_type: - sample_data = {k: v for k, v in sample_data.items() if k != omit_data_type} - - search = { - 'sample_data': sample_data, - 'genome_version': genome_version, - 'num_results': num_results, - **search_body, - } - search.update(search_body or {}) - return search diff --git a/hail_search/web_app.py b/hail_search/web_app.py deleted file mode 100644 index 2954edc322..0000000000 --- a/hail_search/web_app.py +++ /dev/null @@ -1,138 +0,0 @@ -from aiohttp import web -import asyncio -import concurrent.futures -import ctypes -import functools -import json -import os -import hail as hl -import logging -import traceback -from typing import Callable - -from hail_search.search import search_hail_backend, load_globals, lookup_variant, lookup_variants - -logger = logging.getLogger(__name__) - -JAVA_OPTS_XSS = os.environ.get('JAVA_OPTS_XSS') -MACHINE_MEM = os.environ.get('MACHINE_MEM') -JVM_MEMORY_FRACTION = 0.9 -QUERY_TIMEOUT_S = 300 - - -def _handle_exception(e, request): - logger.error(f'{request.headers.get("From")} "{e}"') - raise e - - -@web.middleware -async def error_middleware(request, handler): - try: - return await handler(request) - except web.HTTPError as e: - _handle_exception(e, request) - except Exception as e: - error_reason = f'{e}: {traceback.format_exc()}' - _handle_exception(web.HTTPInternalServerError(reason=error_reason), request) - - -def _hl_json_default(o): - if isinstance(o, hl.Struct) or isinstance(o, hl.utils.frozendict): - return dict(o) - elif isinstance(o, set): - return sorted(o) - - -def hl_json_dumps(obj): - return json.dumps(obj, default=_hl_json_default) - -async def sync_to_async_hail_query(request: web.Request, query: Callable, *args, timeout_s=QUERY_TIMEOUT_S, **kwargs): - request_body = None - if request.body_exists: - request_body = await request.json() - - loop = asyncio.get_running_loop() - future = loop.run_in_executor(request.app.pool, functools.partial(query, request_body, *args, **kwargs)) - try: - return await asyncio.wait_for(future, timeout_s) - except asyncio.TimeoutError: - # Well documented issue with the "wait_for" approach.... the concurrent.Future is canceled but - # the underlying thread is not, allowing the Hail Query under the hood to keep running. - # https://stackoverflow.com/questions/34452590/timeout-handling-while-using-run-in-executor-and-asyncio - # This unsafe approach is taken from: - # https://stackoverflow.com/questions/323972/is-there-any-way-to-kill-a-thread - # - # A few other thoughts: - # - A ProcessPoolExecutor instead of a ThreadPoolExecutor would allow for safe worker termination - # and would potentially be a safer option in general (some portion of a hail query is cpu bound in python!) - # - A "timeout" decorator applied to the query function, catching a SIGALARM would also potentially - # suffice... but threads don't play well with signals. - # - We could also just kill the service/pod (which is fine). - for t in request.app.pool._threads: - res = ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(t.ident), ctypes.py_object(TimeoutError)) - if res > 1: - # "if it returns a number greater than one, you're in trouble, - # and you should call it again with exc=NULL to revert the effect" - ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(t.ident), None) - raise SystemExit('PyThreadState_SetAsyncExc failed') - raise TimeoutError('Hail Query Timeout Exceeded') - -async def gene_counts(request: web.Request) -> web.Response: - hail_results = await sync_to_async_hail_query(request, search_hail_backend, gene_counts=True) - return web.json_response(hail_results, dumps=hl_json_dumps) - - -async def search(request: web.Request) -> web.Response: - hail_results, total_results = await sync_to_async_hail_query(request, search_hail_backend) - return web.json_response({'results': hail_results, 'total': total_results}, dumps=hl_json_dumps) - - -async def lookup(request: web.Request) -> web.Response: - result = await sync_to_async_hail_query(request, lookup_variant) - return web.json_response(result, dumps=hl_json_dumps) - - -async def multi_lookup(request: web.Request) -> web.Response: - result = await sync_to_async_hail_query(request, lookup_variants) - return web.json_response({'results': result}, dumps=hl_json_dumps) - - -async def reload_globals(request: web.Request) -> web.Response: - result = await sync_to_async_hail_query(request, lambda _: load_globals()) - return web.json_response( - result, - dumps=hl_json_dumps - ) - - -async def status(request: web.Request) -> web.Response: - # Make sure the hail backend process is still alive. - await sync_to_async_hail_query(request, lambda _: hl.eval(1 + 1)) - return web.json_response({'success': True}) - - -async def init_web_app(): - spark_conf = {} - # memory limits adapted from https://github.com/hail-is/hail/blob/main/hail/python/hailtop/hailctl/dataproc/start.py#L321C17-L321C36 - if MACHINE_MEM: - spark_conf['spark.driver.memory'] = f'{int((int(MACHINE_MEM)-11)*JVM_MEMORY_FRACTION)}g' - if JAVA_OPTS_XSS: - spark_conf.update({f'spark.{field}.extraJavaOptions': f'-Xss{JAVA_OPTS_XSS}' for field in ['driver', 'executor']}) - hl.init(idempotent=True, spark_conf=spark_conf or None) - hl._set_flags(use_new_shuffle='1') - load_globals() - app = web.Application(middlewares=[error_middleware], client_max_size=(1024**2)*20) - app.add_routes([ - web.get('/status', status), - web.post('/reload_globals', reload_globals), - web.post('/search', search), - web.post('/gene_counts', gene_counts), - web.post('/lookup', lookup), - web.post('/multi_lookup', multi_lookup), - ]) - # The idea here is to run the hail queries off the main thread so that the - # event loop stays live and the /status check is responsive. We only - # run a single thread, though, so that hail queries block hail queries - # and we never run more than a single hail query at a time. - app.pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) - return app From 095a5f404647ac1bd1914e425a3716bd8398d733 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Wed, 10 Sep 2025 14:22:07 -0400 Subject: [PATCH 33/89] Revert "update docs for dataproc + clickhouse" This reverts commit 84848a6e776ad716d2aff4be193073c487bd6acc. --- deploy/LOCAL_INSTALL_HELM.md | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/deploy/LOCAL_INSTALL_HELM.md b/deploy/LOCAL_INSTALL_HELM.md index c217bfd3ac..48056a519d 100644 --- a/deploy/LOCAL_INSTALL_HELM.md +++ b/deploy/LOCAL_INSTALL_HELM.md @@ -95,20 +95,19 @@ kubectl exec seqr-POD-ID -c seqr -it -- bash gcloud auth list ``` This service account must be permissioned to create and modify dataproc clusters, and read and write to the GCS buckets that will host your data. We also recommend verifying that the Cloud API access scopes are permissioned for both IAM and Compute Engine access. -- [Create GCS service account HMAC key id and secret key](https://clickhouse.com/docs/integrations/gcs#generate-an-access-key) for your service account. These keys are used by the direct Clickhouse integration with Google Cloud Storage. -- Set those values as secrets within your Kubernetes envirionment: -``` -kubectl create secret generic clickhouse-hmac-secrets ---from-literal=pipeline_data_dir_access_id='ACCESS_KEY_ID' ---from-literal=pipeline_data_dir_access_key='SECRET_KEY' -``` - Create or add the following environment variables to your existing `helm` [override values file](https://github.com/broadinstitute/seqr-helm?tab=readme-ov-file#valuesenvironment-overrides): ```yaml pipeline-runner: environment: + # Stores authoritative copies of your hail search data. + # Objects should not expire from this bucket. + HAIL_SEARCH_DATA_DIR: 'gs://...' # Bucket used as a scratch directory by HAIL during loading. # We recommend a <1 week expiration policy for this bucket. HAIL_TMP_DIR: 'gs://...' + # Bucket used for caching and worker communcation by the pipeline. + # We recommend a ~2 week expiration policy for this bucket. + LOADING_DATASETS_DIR: 'gs://...' # Stores a copy of seqr's reference data hosted in your project's account. # Objects should not expire from this bucket. REFERENCE_DATASETS_DIR: 'gs://...' @@ -118,11 +117,8 @@ pipeline-runner: GCLOUD_ZONE: 'zone-of-dataproc-cluster' seqr: environment: - # Stores authoritative copies of newly annotated variants - # and sample calls from the pipeline. Objects should not expire from this bucket. - PIPELINE_DATA_DIR: 'gs://...' - # Bucket used for caching and worker communication by the pipeline. - # We recommend a ~2 week expiration policy for this bucket. + # Bucket used for caching and worker communcation by the pipeline. + # Must be identical to `LOADING_DATASETS_DIR` stored above. LOADING_DATASETS_DIR: 'gs://...' ``` - Install/Upgrade with the override values applied: From 5401e6072a4d78e936b9854170237c8e5c02f81c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Sep 2025 14:24:42 -0400 Subject: [PATCH 34/89] clean up docker ignore --- .dockerignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index e1a4f868bb..e294067478 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,9 +5,7 @@ static/* !static/images/* deploy/* !deploy/docker/seqr/* -!deploy/docker/hail_search/* !deploy/docker/vlm/* -hail_search/fixtures/* .git .vscode .idea From 15c3fb8e517cdb762d0781c9c82f5e3e6ea6ec21 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Sep 2025 15:53:38 -0400 Subject: [PATCH 35/89] better labels for intenral vs internal vlm results --- ui/pages/SummaryData/components/VariantLookup.jsx | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ui/pages/SummaryData/components/VariantLookup.jsx b/ui/pages/SummaryData/components/VariantLookup.jsx index d23c82b86d..820bb9691d 100644 --- a/ui/pages/SummaryData/components/VariantLookup.jsx +++ b/ui/pages/SummaryData/components/VariantLookup.jsx @@ -194,10 +194,10 @@ const VariantLookup = ({ queryParams, receiveData, updateQueryParams, vlmEnabled {vlmEnabled && ( - -
+ +
- + )} + {vlmEnabled && ( + + +
+ + + + )} Date: Wed, 10 Sep 2025 17:39:29 -0400 Subject: [PATCH 36/89] add linting command --- .github/workflows/unit-tests.yml | 2 ++ ui/package.json | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6a8e929bfa..cc339c0049 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -118,5 +118,7 @@ jobs: - run: npm install -g npm@7.24.2 - run: npm ci working-directory: ./ui + - run: npm run lint + working-directory: ./ui - run: npm test working-directory: ./ui diff --git a/ui/package.json b/ui/package.json index 16d07b045a..9d9dabe99d 100644 --- a/ui/package.json +++ b/ui/package.json @@ -121,6 +121,7 @@ "start": "node scripts/start.js", "build": "node scripts/build.js", "test": "node scripts/test.js --env=jsdom", + "lint": "eslint --ext .js,.jsx --ignore-pattern \"**/*.test.js\" --ignore-pattern \"**/fixtures.js\" --ignore-pattern \"scripts/*\" .", "postinstall": "patch-package" }, "jest": { From 4ae80f5626528599000e33e648ad20a205104e94 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Sep 2025 17:49:31 -0400 Subject: [PATCH 37/89] remove unused files --- ui/shared/components/form/MessagesPanel.jsx | 46 --------------------- ui/shared/components/page/BreadCrumbs.jsx | 36 ---------------- 2 files changed, 82 deletions(-) delete mode 100644 ui/shared/components/form/MessagesPanel.jsx delete mode 100644 ui/shared/components/page/BreadCrumbs.jsx diff --git a/ui/shared/components/form/MessagesPanel.jsx b/ui/shared/components/form/MessagesPanel.jsx deleted file mode 100644 index 93b4e46f69..0000000000 --- a/ui/shared/components/form/MessagesPanel.jsx +++ /dev/null @@ -1,46 +0,0 @@ -import React from 'react' -import PropTypes from 'prop-types' -import styled from 'styled-components' - -import { Message } from 'semantic-ui-react' - -const MessagePanelContainer = styled.div` - padding: 5px 10px; -` - -const StyledMessage = styled(Message)` - margin: 10px 0px 30px 0px!important; - display: block !important; - text-align: left !important; -` - -const MessagesPanel = React.memo(({ info, warnings, errors }) => - - { - info && info.length > 0 && - - {info.map(infoItem =>
{infoItem}
)} - - } - { - warnings && warnings.length > 0 && - - {warnings.map(warning =>
WARNING: {warning}
)} - - } - { - errors && errors.length > 0 && - - {errors.map(error =>
ERROR: {error}
)} - - } - , -) - -MessagesPanel.propTypes = { - errors: PropTypes.array, - warnings: PropTypes.array, - info: PropTypes.array, -} - -export default MessagesPanel diff --git a/ui/shared/components/page/BreadCrumbs.jsx b/ui/shared/components/page/BreadCrumbs.jsx deleted file mode 100644 index d342dcb310..0000000000 --- a/ui/shared/components/page/BreadCrumbs.jsx +++ /dev/null @@ -1,36 +0,0 @@ -/* eslint-disable react/no-array-index-key */ - -import React from 'react' -import PropTypes from 'prop-types' - -import { Breadcrumb } from 'semantic-ui-react' - -const BreadCrumbs = React.memo(props => -
- { - props.breadcrumbSections.map((label, i) => ( - { - i < props.breadcrumbSections.length - 1 ? - ( - - {props.breadcrumbSections[i]} - » - - ) : ( - - {props.breadcrumbSections[i]} - - ) - } - ), - ) - } - -
, -) - -BreadCrumbs.propTypes = { - breadcrumbSections: PropTypes.array.isRequired, -} - -export default BreadCrumbs From 933dce01c6cd0c92fe8f0a9848146d4961ebdd93 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Sep 2025 17:51:46 -0400 Subject: [PATCH 38/89] remov eunused test file --- ui/shared/components/page/BreadCrumbs.test.js | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 ui/shared/components/page/BreadCrumbs.test.js diff --git a/ui/shared/components/page/BreadCrumbs.test.js b/ui/shared/components/page/BreadCrumbs.test.js deleted file mode 100644 index 6ba1fcfeb8..0000000000 --- a/ui/shared/components/page/BreadCrumbs.test.js +++ /dev/null @@ -1,18 +0,0 @@ -import React from 'react' -import { shallow, configure } from 'enzyme' -import Adapter from '@wojtekmaj/enzyme-adapter-react-17' -import BreadCrumbs from './BreadCrumbs' - -configure({ adapter: new Adapter() }) - -test('shallow-render without crashing', () => { - /* - breadcrumbSections: PropTypes.array.isRequired, - */ - - const props = { - breadcrumbSections: ['base', 'subsection', 'subsection2'], - } - - shallow() -}) From f256436c34be56188a66ae4818233977a075dead Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 11 Sep 2025 11:03:49 -0400 Subject: [PATCH 39/89] allow project, family, and matchmaker page loading when clickhouse is down --- matchmaker/views/matchmaker_api.py | 2 +- seqr/views/apis/family_api.py | 12 +++++++++--- seqr/views/utils/orm_to_json_utils.py | 15 +++++++++++++-- seqr/views/utils/project_context_utils.py | 2 +- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/matchmaker/views/matchmaker_api.py b/matchmaker/views/matchmaker_api.py index fa0fa3cf15..bde44a592f 100644 --- a/matchmaker/views/matchmaker_api.py +++ b/matchmaker/views/matchmaker_api.py @@ -47,7 +47,7 @@ def get_individual_mme_matches(request, submission_guid): results = MatchmakerResult.objects.filter(submission=submission) response_json = get_json_for_saved_variants_with_tags( - SavedVariant.objects.filter(family=submission.individual.family), add_details=True, genome_version=project.genome_version) + SavedVariant.objects.filter(family=submission.individual.family), add_details=True, allow_failed_detail=True, genome_version=project.genome_version) gene_ids = set() for variant in response_json['savedVariantsByGuid'].values(): diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index af71b1b34c..c5c96f3e2c 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -12,6 +12,7 @@ from matchmaker.models import MatchmakerSubmission from reference_data.models import Omim from seqr.utils.gene_utils import get_genes_for_variant_display +from seqr.utils.logging_utils import SeqrLogger from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.file_utils import save_uploaded_file, load_uploaded_file from seqr.views.utils.individual_utils import delete_individuals @@ -29,6 +30,8 @@ from seqr.views.utils.variant_utils import get_phenotype_prioritization, get_omim_intervals_query, DISCOVERY_CATEGORY from seqr.utils.xpos_utils import get_chrom_pos +logger = SeqrLogger(__name__) + FAMILY_ID_FIELD = 'familyId' PREVIOUS_FAMILY_ID_FIELD = 'previousFamilyId' @@ -60,7 +63,7 @@ def family_page_data(request, family_guid): ) gene_ids = backend_specific_call( _variants_gene_ids, _variants_gene_ids, _clickhouse_variants_gene_ids, - )(discovery_variants, project.genome_version) + )(discovery_variants, project.genome_version, request.user) discovery_variant_intervals = [dict(zip( ['chrom', 'start', 'end_chrom', 'end', 'svType', 'hasSvType'], [*get_chrom_pos(v['xpos']), *get_chrom_pos(v['xpos_end']), v['svType'], (v.get('dataset_type') or '').startswith(Sample.DATASET_TYPE_SV_CALLS)] @@ -107,7 +110,7 @@ def _variants_gene_ids(variants, *args, **kwargs): return {gene_id for variant in variants for gene_id in (variant['transcripts'] or {}).keys()} -def _clickhouse_variants_gene_ids(variants, genome_version): +def _clickhouse_variants_gene_ids(variants, genome_version, user): keys_by_dataset_type = defaultdict(set) no_key_variants = [] for v in variants: @@ -117,7 +120,10 @@ def _clickhouse_variants_gene_ids(variants, genome_version): no_key_variants.append(v) gene_ids = _variants_gene_ids(no_key_variants) for dataset_type, keys in keys_by_dataset_type.items(): - gene_ids.update(get_clickhouse_genes(genome_version, dataset_type, keys)) + try: + gene_ids.update(get_clickhouse_genes(genome_version, dataset_type, keys)) + except Exception as e: + logger.error(f'Error loading genes from clickhouse: {e}', user) return gene_ids diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 1f6ecb75dc..987c1bfd88 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -15,6 +15,7 @@ from reference_data.models import HumanPhenotypeOntology from seqr.models import GeneNote, VariantNote, VariantTag, VariantFunctionalData, SavedVariant, Family, CAN_VIEW, CAN_EDIT, \ get_audit_field_names, RnaSeqOutlier, RnaSeqSpliceOutlier +from seqr.utils.logging_utils import SeqrLogger from seqr.views.utils.json_utils import _to_camel_case from seqr.views.utils.permissions_utils import has_project_permissions, \ project_has_anvil, get_workspace_collaborator_perms, user_is_analyst, user_is_data_manager, user_is_pm, \ @@ -22,6 +23,8 @@ from seqr.views.utils.terra_api_utils import is_anvil_authenticated, anvil_enabled from settings import ANALYST_USER_GROUP, SERVICE_ACCOUNT_FOR_ANVIL, MEDIA_URL +logger = SeqrLogger(__name__) + def _get_model_json_fields(model_class, user, is_analyst, additional_model_fields): fields = set(model_class._meta.json_fields) @@ -412,7 +415,7 @@ def get_json_for_analysis_group(analysis_group, **kwargs): return _get_json_for_model(analysis_group, get_json_for_models=get_json_for_analysis_groups, **kwargs) -def get_json_for_saved_variants(saved_variants, add_details=False, additional_model_fields=None, additional_values=None, genome_version=None): +def get_json_for_saved_variants(saved_variants, add_details=False, additional_model_fields=None, additional_values=None, genome_version=None, allow_failed_detail=False): sv_additional_values = { 'familyGuids': ArrayAgg('family__guid', distinct=True), } @@ -436,7 +439,8 @@ def get_json_for_saved_variants(saved_variants, add_details=False, additional_mo if add_details: from seqr.utils.search.utils import backend_specific_call - backend_specific_call(lambda *args: None, lambda *args: None, _add_clickhouse_annotations)(results, genome_version) + clickhouse_func = _safe_add_clickhouse_annotations if allow_failed_detail else _add_clickhouse_annotations + backend_specific_call(lambda *args: None, lambda *args: None, clickhouse_func)(results, genome_version) for result in results: result.update({k: v for k, v in result.pop('savedVariantJson').items() if k not in result}) @@ -465,6 +469,13 @@ def _add_clickhouse_annotations(results, genome_version): result.update(annotations_by_key[result['key']]) +def _safe_add_clickhouse_annotations(*args, **kwargs): + try: + _add_clickhouse_annotations(*args, **kwargs) + except Exception as e: + logger.error(f'Error loading saved variant annotations from clickhouse: {e}', None) + + def _format_functional_tags(tags): for tag in tags: name = tag.pop('functionalDataTag') diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index 98319a2392..736d25b316 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -116,7 +116,7 @@ def families_discovery_tags(families, genome_version, project=None): family_filter = {'family__project': project} if project else {'family__guid__in': families_by_guid.keys()} discovery_tags = get_json_for_saved_variants(SavedVariant.objects.filter( varianttag__variant_tag_type__category='CMG Discovery Tags', **family_filter, - ), add_details=True, genome_version=genome_version) + ), add_details=True, allow_failed_detail=True, genome_version=genome_version) gene_ids = set() for tag in discovery_tags: From 702ef2b960033640711995e11efac0d05b8921ea Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 11 Sep 2025 11:45:59 -0400 Subject: [PATCH 40/89] test project page loads without clickhouse --- seqr/views/apis/project_api_tests.py | 65 ++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index 7aee432b5f..d2820fd9b7 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -3,6 +3,7 @@ from copy import deepcopy from datetime import datetime from django.contrib.auth.models import Group +from django.db import connections from django.urls.base import reverse import responses @@ -40,11 +41,39 @@ {'id': 'recSgwrXNkmlIB5eM', 'fields': {'Status': 'Available in Seqr'}}, ]} +DISCOVERY_TAG = { + 'CAID': 'CA1501729', + 'acmgClassification': None, + 'alt': 'T', + 'chrom': '1', + 'clinvar': {'alleleId': 12345, 'assertions': None, 'conditions': None, 'conflictingPathogenicities': None, 'goldStars': None, 'pathogenicity': 'Uncertain_significance', 'submitters': None}, + 'familyGuids': ['F000002_2'], + 'genomeVersion': '37', + 'genotypes': {'I000004_hg00731': {'numAlt': 2}, 'I000005_hg00732': {'numAlt': 1}}, + 'hgmd': mock.ANY, + 'liftedOverChrom': mock.ANY, + 'liftedOverGenomeVersion': '38', + 'liftedOverPos': mock.ANY, + 'populations': mock.ANY, + 'pos': 248367227, + 'predictions': mock.ANY, + 'ref': 'TC', + 'rsid': None, + 'selectedMainTranscriptId': 'ENST00000371839', + 'transcripts': { + 'ENSG00000135953': mock.ANY, + 'ENSG00000240361': mock.ANY, + }, + 'variantGuid': 'SV0000002_1248367227_r0390_100', + 'variantId': '1-248367227-TC-T', + 'xpos': 1248367227, +} class ProjectAPITest(object): CREATE_PROJECT_JSON = WORKSPACE_CREATE_PROJECT_JSON REQUIRED_FIELDS = ['name', 'genomeVersion', 'workspaceNamespace', 'workspaceName'] AIRTABLE_TRACKING_URL = f'{MOCK_AIRTABLE_URL}/appUelDNM3BnWaR7M/AnVIL%20Seqr%20Loading%20Requests%20Tracking' + DISCOVERY_TAG = DISCOVERY_TAG @mock.patch('seqr.views.utils.airtable_utils.logger') @mock.patch('seqr.views.utils.airtable_utils.AIRTABLE_URL', MOCK_AIRTABLE_URL) @@ -372,11 +401,19 @@ def test_project_families(self): url = reverse(project_families, args=[PROJECT_GUID]) self.check_collaborator_login(url) + response_keys = {'familiesByGuid', 'genesById'} + + empty_url = reverse(project_families, args=[EMPTY_PROJECT_GUID]) + self._check_empty_project(empty_url, response_keys) + + gene_ids = self._assert_expected_project_families(url, response_keys) + self.assertSetEqual(gene_ids, {'ENSG00000135953', 'ENSG00000240361'}) + + def _assert_expected_project_families(self, url, response_keys): response = self.client.get(url) self.assertEqual(response.status_code, 200) response_json = response.json() - response_keys = {'familiesByGuid', 'genesById'} self.assertSetEqual(set(response_json.keys()), response_keys) family_1 = response_json['familiesByGuid']['F000001_1'] @@ -419,19 +456,13 @@ def test_project_families(self): self.assertListEqual(family_3['discoveryTags'], []) self.assertListEqual(empty_family['discoveryTags'], []) self.assertSetEqual({tag['variantGuid'] for tag in family_1['discoveryTags']}, {'SV0000001_2103343353_r0390_100'}) - self.assertSetEqual( - {tag['variantGuid'] for tag in response_json['familiesByGuid']['F000002_2']['discoveryTags']}, - {'SV0000002_1248367227_r0390_100'}) + self.assertListEqual(response_json['familiesByGuid']['F000002_2']['discoveryTags'], [self.DISCOVERY_TAG]) no_discovery_families = set(response_json['familiesByGuid'].keys()) - {'F000001_1', 'F000002_2'} self.assertSetEqual({ len(response_json['familiesByGuid'][family_guid]['discoveryTags']) for family_guid in no_discovery_families }, {0}) - self.assertSetEqual(set(response_json['genesById'].keys()), {'ENSG00000135953', 'ENSG00000240361'}) - - # Test empty project - empty_url = reverse(project_families, args=[EMPTY_PROJECT_GUID]) - self._check_empty_project(empty_url, response_keys) + return set(response_json['genesById'].keys()) def test_project_individuals(self): url = reverse(project_individuals, args=[PROJECT_GUID]) @@ -704,6 +735,7 @@ class AnvilProjectAPITest(AnvilAuthenticationTestCase, ProjectAPITest): PROJECT_COLLABORATORS = ANVIL_COLLABORATORS PROJECT_COLLABORATOR_GROUPS = None HAS_EMPTY_PROJECT = False + DISCOVERY_TAG = {**DISCOVERY_TAG, 'key': 100, 'mainTranscriptId': 'ENST00000505820'} def test_create_and_delete_project(self, *args, **kwargs): super(AnvilProjectAPITest, self).test_create_and_delete_project(*args, **kwargs) @@ -777,3 +809,18 @@ def test_project_collaborators(self): 'my-seqr-billing', 'anvil-1kg project n\u00e5me with uni\u00e7\u00f8de') self.assertEqual(self.mock_get_ws_access_level.call_count, 5) + + def _assert_expected_project_families(self, *args, **kwargs): + gene_ids = super()._assert_expected_project_families(*args, **kwargs) + + # Test success when clickhouse is unavailable + connections['clickhouse'].close() + self.DISCOVERY_TAG = {k: v for k, v in self.DISCOVERY_TAG.items() if k in { + 'acmgClassification', 'alt', 'familyGuids', 'genotypes', 'key', 'ref', 'selectedMainTranscriptId', + 'variantGuid', 'variantId', 'xpos', + }} + no_clickhouse_gene_ids = super()._assert_expected_project_families(*args, **kwargs) + self.assertSetEqual(no_clickhouse_gene_ids, {'ENSG00000135953'}) + + return gene_ids + From 8e90faf8a7a164b92e33e78b570eda46ce933352 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 11 Sep 2025 11:54:50 -0400 Subject: [PATCH 41/89] assert valid alerting --- seqr/views/apis/project_api_tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index d2820fd9b7..832a641f34 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -814,6 +814,7 @@ def _assert_expected_project_families(self, *args, **kwargs): gene_ids = super()._assert_expected_project_families(*args, **kwargs) # Test success when clickhouse is unavailable + self.reset_logs() connections['clickhouse'].close() self.DISCOVERY_TAG = {k: v for k, v in self.DISCOVERY_TAG.items() if k in { 'acmgClassification', 'alt', 'familyGuids', 'genotypes', 'key', 'ref', 'selectedMainTranscriptId', @@ -821,6 +822,12 @@ def _assert_expected_project_families(self, *args, **kwargs): }} no_clickhouse_gene_ids = super()._assert_expected_project_families(*args, **kwargs) self.assertSetEqual(no_clickhouse_gene_ids, {'ENSG00000135953'}) + self.assert_json_logs(None, [ + ("Error loading saved variant annotations from clickhouse: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.", { + 'severity': 'ERROR', + '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', + }), + ]) return gene_ids From 3d80c31416c53c4642989d3d61b00f344da906f3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 11 Sep 2025 12:30:59 -0400 Subject: [PATCH 42/89] get minimal set of transcript fields --- seqr/views/utils/project_context_utils.py | 32 ++++++++++++++++++----- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index 736d25b316..1cf792c8b1 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -1,12 +1,14 @@ from collections import defaultdict from django.db.models import Count, Q, F, prefetch_related_objects +from clickhouse_search.search import get_transcripts_queryset from seqr.models import Individual, IgvSample, AnalysisGroup, DynamicAnalysisGroup, LocusList, VariantTagType,\ VariantFunctionalData, FamilyNote, SavedVariant, VariantTag, VariantNote from seqr.utils.gene_utils import get_genes +from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.orm_to_json_utils import _get_json_for_families, _get_json_for_individuals, get_json_for_queryset, \ get_json_for_analysis_groups, get_json_for_samples, get_json_for_locus_lists, \ - get_json_for_family_notes, get_json_for_saved_variants + get_json_for_family_notes def get_projects_child_entities(projects, project_guid, user): @@ -114,15 +116,20 @@ def families_discovery_tags(families, genome_version, project=None): families_by_guid = {f['familyGuid']: dict(discoveryTags=[], **f) for f in families} family_filter = {'family__project': project} if project else {'family__guid__in': families_by_guid.keys()} - discovery_tags = get_json_for_saved_variants(SavedVariant.objects.filter( + discovery_tags_by_key = {v.pop('key'): v for v in SavedVariant.objects.filter( varianttag__variant_tag_type__category='CMG Discovery Tags', **family_filter, - ), add_details=True, allow_failed_detail=True, genome_version=genome_version) + ).values( + 'key', 'family__guid', selectedMainTranscriptId=F('selected_main_transcript_id'), + transcripts=F('saved_variant_json__transcripts'), mainTranscriptId=F('saved_variant_json__mainTranscriptId'), + )} + backend_specific_call( + lambda *args: None, lambda *args: None, _add_clickhouse_transcripts, + )(discovery_tags_by_key, genome_version) gene_ids = set() - for tag in discovery_tags: + for tag in discovery_tags_by_key.values(): gene_ids.update(list(tag.get('transcripts', {}).keys())) - for family_guid in tag['familyGuids']: - families_by_guid[family_guid]['discoveryTags'].append(tag) + families_by_guid[tag.pop('family__guid')]['discoveryTags'].append(tag) return { 'familiesByGuid': families_by_guid, @@ -130,6 +137,19 @@ def families_discovery_tags(families, genome_version, project=None): } +def _add_clickhouse_transcripts(discovery_tags_by_key, genome_version): + transcripts_by_key = dict( + get_transcripts_queryset(genome_version, discovery_tags_by_key.keys()).values_list('key', 'transcripts') + ) + for key, tag in discovery_tags_by_key.items(): + if key in transcripts_by_key: + tag['transcripts'] = transcripts_by_key[key] + tag['mainTranscriptId'] = next( + t['transcriptId'] for gene_transcripts in tag['transcripts'].values() for t in gene_transcripts + if t['transcriptRank'] == 0 + ) + + MME_TAG_NAME = 'MME Submission' From c99c2dca63005b5995eae7a88c5771e063a852d4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 11 Sep 2025 12:42:42 -0400 Subject: [PATCH 43/89] update test --- seqr/views/apis/project_api_tests.py | 37 ++++++----------------- seqr/views/utils/orm_to_json_utils.py | 1 + seqr/views/utils/project_context_utils.py | 13 +++++--- 3 files changed, 20 insertions(+), 31 deletions(-) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index 832a641f34..796a445bee 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -42,31 +42,12 @@ ]} DISCOVERY_TAG = { - 'CAID': 'CA1501729', - 'acmgClassification': None, - 'alt': 'T', - 'chrom': '1', - 'clinvar': {'alleleId': 12345, 'assertions': None, 'conditions': None, 'conflictingPathogenicities': None, 'goldStars': None, 'pathogenicity': 'Uncertain_significance', 'submitters': None}, - 'familyGuids': ['F000002_2'], - 'genomeVersion': '37', - 'genotypes': {'I000004_hg00731': {'numAlt': 2}, 'I000005_hg00732': {'numAlt': 1}}, - 'hgmd': mock.ANY, - 'liftedOverChrom': mock.ANY, - 'liftedOverGenomeVersion': '38', - 'liftedOverPos': mock.ANY, - 'populations': mock.ANY, - 'pos': 248367227, - 'predictions': mock.ANY, - 'ref': 'TC', - 'rsid': None, + 'mainTranscriptId': None, 'selectedMainTranscriptId': 'ENST00000371839', 'transcripts': { 'ENSG00000135953': mock.ANY, 'ENSG00000240361': mock.ANY, }, - 'variantGuid': 'SV0000002_1248367227_r0390_100', - 'variantId': '1-248367227-TC-T', - 'xpos': 1248367227, } class ProjectAPITest(object): @@ -455,7 +436,12 @@ def _assert_expected_project_families(self, url, response_keys): self.assertListEqual(family_3['discoveryTags'], []) self.assertListEqual(empty_family['discoveryTags'], []) - self.assertSetEqual({tag['variantGuid'] for tag in family_1['discoveryTags']}, {'SV0000001_2103343353_r0390_100'}) + self.maxDiff = None + self.assertListEqual(family_1['discoveryTags'], [{ + 'transcripts': {'ENSG00000135953': [mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY]}, + 'mainTranscriptId': 'ENST00000258436', + 'selectedMainTranscriptId': None, + }]) self.assertListEqual(response_json['familiesByGuid']['F000002_2']['discoveryTags'], [self.DISCOVERY_TAG]) no_discovery_families = set(response_json['familiesByGuid'].keys()) - {'F000001_1', 'F000002_2'} self.assertSetEqual({ @@ -735,7 +721,7 @@ class AnvilProjectAPITest(AnvilAuthenticationTestCase, ProjectAPITest): PROJECT_COLLABORATORS = ANVIL_COLLABORATORS PROJECT_COLLABORATOR_GROUPS = None HAS_EMPTY_PROJECT = False - DISCOVERY_TAG = {**DISCOVERY_TAG, 'key': 100, 'mainTranscriptId': 'ENST00000505820'} + DISCOVERY_TAG = {**DISCOVERY_TAG, 'mainTranscriptId': 'ENST00000505820'} def test_create_and_delete_project(self, *args, **kwargs): super(AnvilProjectAPITest, self).test_create_and_delete_project(*args, **kwargs) @@ -816,14 +802,11 @@ def _assert_expected_project_families(self, *args, **kwargs): # Test success when clickhouse is unavailable self.reset_logs() connections['clickhouse'].close() - self.DISCOVERY_TAG = {k: v for k, v in self.DISCOVERY_TAG.items() if k in { - 'acmgClassification', 'alt', 'familyGuids', 'genotypes', 'key', 'ref', 'selectedMainTranscriptId', - 'variantGuid', 'variantId', 'xpos', - }} + self.DISCOVERY_TAG = {**DISCOVERY_TAG, 'transcripts': {}} no_clickhouse_gene_ids = super()._assert_expected_project_families(*args, **kwargs) self.assertSetEqual(no_clickhouse_gene_ids, {'ENSG00000135953'}) self.assert_json_logs(None, [ - ("Error loading saved variant annotations from clickhouse: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.", { + ("Error loading discovery genes from clickhouse: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.", { 'severity': 'ERROR', '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', }), diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 987c1bfd88..5ecf22688e 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -469,6 +469,7 @@ def _add_clickhouse_annotations(results, genome_version): result.update(annotations_by_key[result['key']]) +# TODO revert? def _safe_add_clickhouse_annotations(*args, **kwargs): try: _add_clickhouse_annotations(*args, **kwargs) diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index 1cf792c8b1..b6a4aa3740 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -5,11 +5,14 @@ from seqr.models import Individual, IgvSample, AnalysisGroup, DynamicAnalysisGroup, LocusList, VariantTagType,\ VariantFunctionalData, FamilyNote, SavedVariant, VariantTag, VariantNote from seqr.utils.gene_utils import get_genes +from seqr.utils.logging_utils import SeqrLogger from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.orm_to_json_utils import _get_json_for_families, _get_json_for_individuals, get_json_for_queryset, \ get_json_for_analysis_groups, get_json_for_samples, get_json_for_locus_lists, \ get_json_for_family_notes +logger = SeqrLogger(__name__) + def get_projects_child_entities(projects, project_guid, user): projects_by_guid = {p.guid: {'projectGuid': p.guid, 'name': p.name} for p in projects} @@ -122,13 +125,15 @@ def families_discovery_tags(families, genome_version, project=None): 'key', 'family__guid', selectedMainTranscriptId=F('selected_main_transcript_id'), transcripts=F('saved_variant_json__transcripts'), mainTranscriptId=F('saved_variant_json__mainTranscriptId'), )} - backend_specific_call( - lambda *args: None, lambda *args: None, _add_clickhouse_transcripts, - )(discovery_tags_by_key, genome_version) + try: + backend_specific_call(lambda *args: None, lambda *args: None, _add_clickhouse_transcripts)(discovery_tags_by_key, genome_version) + except Exception as e: + logger.error(f'Error loading discovery genes from clickhouse: {e}', None) gene_ids = set() for tag in discovery_tags_by_key.values(): - gene_ids.update(list(tag.get('transcripts', {}).keys())) + tag['transcripts'] = tag.get('transcripts') or {} + gene_ids.update(list(tag['transcripts'].keys())) families_by_guid[tag.pop('family__guid')]['discoveryTags'].append(tag) return { From b0ee56b2161f8d46e21e444d25d2e7a7715abb4b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 11 Sep 2025 12:45:13 -0400 Subject: [PATCH 44/89] fix test --- seqr/views/apis/family_api_tests.py | 6 +++++- seqr/views/apis/project_api_tests.py | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index 17a1dd9a7e..e6aeb38c82 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -198,7 +198,11 @@ def test_family_variant_tag_summary(self): family = response_json['familiesByGuid'][FAMILY_GUID] self.assertSetEqual(set(family.keys()), {'familyGuid', 'discoveryTags'}) - self.assertSetEqual({tag['variantGuid'] for tag in family['discoveryTags']}, {'SV0000001_2103343353_r0390_100'}) + self.assertListEqual(family['discoveryTags'], [{ + 'transcripts': {'ENSG00000135953': [mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY]}, + 'mainTranscriptId': 'ENST00000258436', + 'selectedMainTranscriptId': None, + }]) project = response_json['projectsByGuid'][PROJECT_GUID] self.assertSetEqual(set(project.keys()), {'variantTagTypes', 'variantFunctionalTagTypes'}) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index 796a445bee..b896e2368e 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -436,7 +436,6 @@ def _assert_expected_project_families(self, url, response_keys): self.assertListEqual(family_3['discoveryTags'], []) self.assertListEqual(empty_family['discoveryTags'], []) - self.maxDiff = None self.assertListEqual(family_1['discoveryTags'], [{ 'transcripts': {'ENSG00000135953': [mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY]}, 'mainTranscriptId': 'ENST00000258436', From 32907afe95752f8fadd4549c84a0067a6645ec88 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 11 Sep 2025 13:54:02 -0400 Subject: [PATCH 45/89] test omim options when no lickhouse --- seqr/views/apis/family_api_tests.py | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index e6aeb38c82..19c9b98f24 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -4,6 +4,7 @@ from copy import deepcopy from datetime import datetime from django.core.files.uploadedfile import SimpleUploadedFile +from django.db import connections from django.urls.base import reverse from matchmaker.models import MatchmakerSubmission @@ -183,6 +184,39 @@ def test_family_page_data(self): response = self.client.get(url.replace(FAMILY_GUID, 'invalid_guid')) self.assertEqual(response.status_code, 404) + # Test family with clickhouse saved variants + omim_options = { + '615120': {'phenotypeMimNumber': 615120, 'phenotypes': [{ + 'geneSymbol': 'RP11', 'mimNumber': 103320, 'phenotypeMimNumber': 615120, + 'phenotypeDescription': 'Myasthenic syndrome, congenital, 8, with pre- and postsynaptic defects', + 'phenotypeInheritance': 'Autosomal recessive, X-linked recessive', 'chrom': '1', 'start': 29554, + 'end': 31109, + }]}, + '616126': {'phenotypeMimNumber': 616126, 'phenotypes': [{ + 'geneSymbol': 'OR4G11P', 'mimNumber': 147571, 'phenotypeMimNumber': 616126, + 'phenotypeDescription': 'Immunodeficiency 38', + 'phenotypeInheritance': 'Autosomal recessive', 'chrom': '1', 'start': 11869, + 'end': 14409, + }]}, + } + self._assert_expected_family2_data(omim_options, url.replace(FAMILY_GUID, FAMILY_GUID2), response_keys, family_fields) + + def _assert_expected_family2_data(self, omim_options, family_2_url, response_keys, family_fields): + response = self.client.get(family_2_url) + self.assertEqual(response.status_code, 200) + response_json = response.json() + self.assertSetEqual(set(response_json.keys()), response_keys) + family = response_json['familiesByGuid'][FAMILY_GUID2] + self.assertSetEqual(set(family.keys()), family_fields) + self.assertDictEqual(family['postDiscoveryOmimOptions'], omim_options) + self.assertEqual(family['projectGuid'], PROJECT_GUID) + self.assertSetEqual(set(family['individualGuids']), set(response_json['individualsByGuid'].keys())) + self.assertEqual(len(response_json['individualsByGuid']), 3) + self.assertEqual(len(response_json['samplesByGuid']), 7) + self.assertEqual(len(response_json['igvSamplesByGuid']), 0) + self.assertEqual(len(response_json['mmeSubmissionsByGuid']), 0) + self.assertEqual(len(response_json['familyNotesByGuid']), 0) + def test_family_variant_tag_summary(self): url = reverse(family_variant_tag_summary, args=[FAMILY_GUID]) self.check_collaborator_login(url) @@ -673,3 +707,17 @@ class AnvilFamilyAPITest(AnvilAuthenticationTestCase, FamilyAPITest): fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_saved_variants'] EXTERNAL_ANVIL_CAN_DELETE = True + + def _assert_expected_family2_data(self, omim_options, *args, **kwargs): + super()._assert_expected_family2_data(omim_options, *args, **kwargs) + + # test graceful handling when clickhouse is down + self.reset_logs() + connections['clickhouse'].close() + super()._assert_expected_family2_data({}, *args, **kwargs) + self.assert_json_logs(self.analyst_user, [ + ("Error loading genes from clickhouse: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.", { + 'severity': 'ERROR', + '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', + }), + ]) From ff986d916e9e8682524f9877e9615c2bfc7c2366 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 11 Sep 2025 15:06:55 -0400 Subject: [PATCH 46/89] actually check connection is live in status check --- seqr/views/status.py | 3 ++- seqr/views/status_tests.py | 54 +++++++++++++++++++++++++------------- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/seqr/views/status.py b/seqr/views/status.py index 4775d94444..d0e6502a1e 100644 --- a/seqr/views/status.py +++ b/seqr/views/status.py @@ -17,7 +17,8 @@ def status_view(request): # Test database connection for db_connection_key in DATABASES.keys(): try: - connections[db_connection_key].cursor() + with connections[db_connection_key].cursor() as cursor: + cursor.execute('SELECT 1') except Exception as e: dependent_services_ok = False logger.error('Database "{}" connection error: {}'.format(db_connection_key, e)) diff --git a/seqr/views/status_tests.py b/seqr/views/status_tests.py index f96464d0ca..8930b8a85d 100644 --- a/seqr/views/status_tests.py +++ b/seqr/views/status_tests.py @@ -1,3 +1,4 @@ +from django.db import connections from django.test import TestCase from django.urls.base import reverse import mock @@ -8,42 +9,49 @@ @mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') +@mock.patch('seqr.views.status.redis.StrictRedis') +@mock.patch('seqr.views.status.logger') class StatusTest(object): - def _test_status_error(self, url, mock_logger): + def teardown(self): + for conn in connections.all(): + conn.connection = None + conn.ensure_connection() + if hasattr(conn.connection, 'pool'): + conn.connection.pool.closed = False + conn.connection.is_closed = False + + @urllib3_responses.activate + def test_status_error(self, mock_logger, mock_redis): + for conn in connections.all(): + conn.ensure_connection() + conn.connection.close() + mock_redis.return_value.ping.side_effect = HTTPError('Bad connection') + urllib3_responses.reset() + urllib3_responses.add(urllib3_responses.HEAD, '/status', status=400) + + url = reverse(status_view) response = self.client.get(url) self.assertEqual(response.status_code, 400) self.assertDictEqual( response.json(), {'version': 'v1.0', 'dependent_services_ok': False, 'secondary_services_ok': False}) calls = [ - mock.call('Database "default" connection error: No connection'), - mock.call('Database "reference_data" connection error: No connection'), - mock.call('Database "clickhouse_write" connection error: No connection'), - mock.call('Database "clickhouse" connection error: No connection'), + mock.call('Database "default" connection error: the connection is closed'), + mock.call('Database "reference_data" connection error: the connection is closed'), + mock.call('Database "clickhouse_write" connection error: connection already closed'), + mock.call('Database "clickhouse" connection error: connection already closed'), mock.call('Redis connection error: Bad connection'), mock.call(f'Search backend connection error: {self.SEARCH_BACKEND_ERROR}'), ] if self.HAS_KIBANA: calls.append(mock.call('Search Admin connection error: Kibana Error 400: Bad Request')) mock_logger.error.assert_has_calls(calls) - mock_logger.reset_mock() - @mock.patch('seqr.views.status.redis.StrictRedis') - @mock.patch('seqr.views.status.connections') - @mock.patch('seqr.views.status.logger') @urllib3_responses.activate - def test_status(self, mock_logger, mock_db_connections, mock_redis): + def test_status(self, mock_logger, mock_redis): url = reverse(status_view) - mock_db_connections.__getitem__.return_value.cursor.side_effect = Exception('No connection') - mock_redis.return_value.ping.side_effect = HTTPError('Bad connection') - urllib3_responses.add(urllib3_responses.HEAD, '/status', status=400) - - self._test_status_error(url, mock_logger) - - mock_db_connections.__getitem__.return_value.cursor.side_effect = None mock_redis.return_value.ping.side_effect = None - urllib3_responses.reset() urllib3_responses.add(urllib3_responses.HEAD, '/', status=200) urllib3_responses.add(urllib3_responses.HEAD, '/status', status=500 if self.HAS_KIBANA else 200) @@ -69,6 +77,7 @@ def test_status(self, mock_logger, mock_db_connections, mock_redis): class ElasticsearchStatusTest(TestCase, StatusTest): + databases = '__all__' SEARCH_BACKEND_ERROR = 'No response from elasticsearch ping' HAS_KIBANA = True @@ -80,8 +89,13 @@ def test_status(self, *args): def _assert_expected_requests(self): self.assertListEqual([call.request.url for call in urllib3_responses.calls], ['/', '/status', '/', '/status']) + def _post_teardown(self): + self.teardown() + super()._post_teardown() + class HailSearchStatusTest(TestCase, StatusTest): + databases = '__all__' SEARCH_BACKEND_ERROR = '400: Bad Request' HAS_KIBANA = False @@ -94,3 +108,7 @@ def test_status(self, *args): def _assert_expected_requests(self): self.assertEqual(len(urllib3_responses.calls), 1) self.assertEqual(urllib3_responses.calls[0].request.url, '/status') + + def _post_teardown(self): + self.teardown() + super()._post_teardown() From b22f301ded93174ab5320bbcbc754c4d75e89712 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 11 Sep 2025 16:05:57 -0400 Subject: [PATCH 47/89] minimal response for mathcmaker saved variants --- clickhouse_search/search.py | 10 ++++--- matchmaker/views/matchmaker_api.py | 32 ++++++++++++++++++++--- seqr/views/utils/orm_to_json_utils.py | 4 +-- seqr/views/utils/project_context_utils.py | 6 ++--- ui/pages/Project/reducers.js | 4 --- 5 files changed, 40 insertions(+), 16 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 1d09cd23cc..c0be75efe6 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -272,13 +272,17 @@ def get_transcripts_queryset(genome_version, keys): return TRANSCRIPTS_CLASS_MAP[genome_version].objects.filter(key__in=keys) +def get_transcripts_by_key(genome_version, keys): + return dict( + get_transcripts_queryset(genome_version, keys_with_transcripts).values_list('key', 'transcripts') + ) + + def format_clickhouse_results(results, genome_version, **kwargs): keys_with_transcripts = { variant['key'] for result in results for variant in (result if isinstance(result, list) else [result]) if not 'transcripts' in variant } - transcripts_by_key = dict( - get_transcripts_queryset(genome_version, keys_with_transcripts).values_list('key', 'transcripts') - ) + transcripts_by_key = get_transcripts_by_key(genome_version, keys_with_transcripts) formatted_results = [] for variant in results: diff --git a/matchmaker/views/matchmaker_api.py b/matchmaker/views/matchmaker_api.py index bde44a592f..a29a727748 100644 --- a/matchmaker/views/matchmaker_api.py +++ b/matchmaker/views/matchmaker_api.py @@ -2,8 +2,10 @@ import requests from datetime import datetime from django.core.mail.message import EmailMessage -from django.db.models import prefetch_related_objects, Q +from django.db.models import prefetch_related_objects, Q, F, Value, CharField +from django.db.models.functions import Coalesce +from clickhouse_search.search import get_transcripts_by_key from matchmaker.models import MatchmakerResult, MatchmakerContactNotes, MatchmakerSubmission, MatchmakerSubmissionGenes, \ MatchmakerIncomingQuery from matchmaker.matchmaker_utils import get_mme_genes_phenotypes_for_results, parse_mme_patient, \ @@ -13,6 +15,8 @@ from seqr.utils.communication_utils import safe_post_to_slack from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException +from seqr.utils.search.utils import backend_specific_call +from seqr.utils.xpos_utils import get_chrom_pos from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json from seqr.views.utils.json_utils import create_json_response @@ -47,10 +51,21 @@ def get_individual_mme_matches(request, submission_guid): results = MatchmakerResult.objects.filter(submission=submission) response_json = get_json_for_saved_variants_with_tags( - SavedVariant.objects.filter(family=submission.individual.family), add_details=True, allow_failed_detail=True, genome_version=project.genome_version) + SavedVariant.objects.filter(family=submission.individual.family), genome_version=project.genome_version, additional_values={ + 'genomeVersion': Coalesce('saved_variant_json__genomeVersion', Value(project.genome_version), output_field=CharField()), + 'transcripts': F('saved_variant_json__transcripts'), + }, additional_model_fields=backend_specific_call([], [], ['key', 'genotypes']), + ) + + variants = response_json['savedVariantsByGuid'].values() + backend_specific_call(lambda *args: None, lambda *args: None, _add_clickhouse_transcripts)( + variants, project.genome_version, request.user, + ) gene_ids = set() - for variant in response_json['savedVariantsByGuid'].values(): + for variant in variants: + chrom, pos = get_chrom_pos(variant['xpos']) + variant.update({'chrom': chrom, 'pos': pos}) gene_ids.update(list(variant.get('transcripts', {}).keys())) hpo_terms_by_id = get_hpo_terms_by_id( @@ -62,6 +77,17 @@ def get_individual_mme_matches(request, submission_guid): submission, results, request.user, additional_genes=gene_ids, response_json=response_json) +def _add_clickhouse_transcripts(variants, genome_version, user): + keys = {variant['key'] for variant in variants} + try: + transcripts_by_key = get_transcripts_by_key(genome_version, keys) + except Exception as e: + logger.error(f'Error loading transcripts from clickhouse: {e}', user) + transcripts_by_key = {} + for variant in variants: + variant['transcripts'] = transcripts_by_key.get(variant['key'], {}) + + @login_and_policies_required def get_mme_nodes(request): return create_json_response({'mmeNodes': list(MME_NODES_BY_NAME.keys())}) diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 5ecf22688e..096136059d 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -536,10 +536,10 @@ def get_json_for_saved_variants_child_entities(tag_cls, saved_variant_id_map, ta return tags, variant_tag_map -def get_json_for_saved_variants_with_tags(saved_variants, **kwargs): +def get_json_for_saved_variants_with_tags(saved_variants, additional_model_fields=None, **kwargs): variants_by_guid = { variant['variantGuid']: dict(tagGuids=[], functionalDataGuids=[], noteGuids=[], **variant) - for variant in get_json_for_saved_variants(saved_variants, additional_model_fields=['id'], **kwargs) + for variant in get_json_for_saved_variants(saved_variants, additional_model_fields=(additional_model_fields or []) + ['id'], **kwargs) } saved_variant_id_map = {} diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index b6a4aa3740..ddbf6b6fbc 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -1,7 +1,7 @@ from collections import defaultdict from django.db.models import Count, Q, F, prefetch_related_objects -from clickhouse_search.search import get_transcripts_queryset +from clickhouse_search.search import get_transcripts_by_key from seqr.models import Individual, IgvSample, AnalysisGroup, DynamicAnalysisGroup, LocusList, VariantTagType,\ VariantFunctionalData, FamilyNote, SavedVariant, VariantTag, VariantNote from seqr.utils.gene_utils import get_genes @@ -143,9 +143,7 @@ def families_discovery_tags(families, genome_version, project=None): def _add_clickhouse_transcripts(discovery_tags_by_key, genome_version): - transcripts_by_key = dict( - get_transcripts_queryset(genome_version, discovery_tags_by_key.keys()).values_list('key', 'transcripts') - ) + transcripts_by_key = get_transcripts_by_key(genome_version, discovery_tags_by_key.keys()) for key, tag in discovery_tags_by_key.items(): if key in transcripts_by_key: tag['transcripts'] = transcripts_by_key[key] diff --git a/ui/pages/Project/reducers.js b/ui/pages/Project/reducers.js index d77a5a923c..9f783e1b98 100644 --- a/ui/pages/Project/reducers.js +++ b/ui/pages/Project/reducers.js @@ -256,13 +256,9 @@ export const getMmeMatches = submissionGuid => (dispatch, getState) => { const state = getState() const submission = state.mmeSubmissionsByGuid[submissionGuid] if (submission && !submission.geneVariants) { - const { familyGuid } = state.individualsByGuid[submission.individualGuid] dispatch({ type: REQUEST_MME_MATCHES }) new HttpRequestHelper(`/api/matchmaker/get_mme_matches/${submissionGuid}`, (responseJson) => { - dispatch({ - type: RECEIVE_SAVED_VARIANT_FAMILIES, updates: { [familyGuid]: { loaded: true, noteVariants: true } }, - }) dispatch({ type: RECEIVE_DATA, updatesById: responseJson, From 66fd793f13a5b8cfbec86c607401c2045bb97246 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 11 Sep 2025 16:12:26 -0400 Subject: [PATCH 48/89] undo helper changes --- seqr/views/utils/orm_to_json_utils.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 096136059d..3cacbc1232 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -15,7 +15,6 @@ from reference_data.models import HumanPhenotypeOntology from seqr.models import GeneNote, VariantNote, VariantTag, VariantFunctionalData, SavedVariant, Family, CAN_VIEW, CAN_EDIT, \ get_audit_field_names, RnaSeqOutlier, RnaSeqSpliceOutlier -from seqr.utils.logging_utils import SeqrLogger from seqr.views.utils.json_utils import _to_camel_case from seqr.views.utils.permissions_utils import has_project_permissions, \ project_has_anvil, get_workspace_collaborator_perms, user_is_analyst, user_is_data_manager, user_is_pm, \ @@ -23,8 +22,6 @@ from seqr.views.utils.terra_api_utils import is_anvil_authenticated, anvil_enabled from settings import ANALYST_USER_GROUP, SERVICE_ACCOUNT_FOR_ANVIL, MEDIA_URL -logger = SeqrLogger(__name__) - def _get_model_json_fields(model_class, user, is_analyst, additional_model_fields): fields = set(model_class._meta.json_fields) @@ -415,7 +412,7 @@ def get_json_for_analysis_group(analysis_group, **kwargs): return _get_json_for_model(analysis_group, get_json_for_models=get_json_for_analysis_groups, **kwargs) -def get_json_for_saved_variants(saved_variants, add_details=False, additional_model_fields=None, additional_values=None, genome_version=None, allow_failed_detail=False): +def get_json_for_saved_variants(saved_variants, add_details=False, additional_model_fields=None, additional_values=None, genome_version=None): sv_additional_values = { 'familyGuids': ArrayAgg('family__guid', distinct=True), } @@ -439,8 +436,7 @@ def get_json_for_saved_variants(saved_variants, add_details=False, additional_mo if add_details: from seqr.utils.search.utils import backend_specific_call - clickhouse_func = _safe_add_clickhouse_annotations if allow_failed_detail else _add_clickhouse_annotations - backend_specific_call(lambda *args: None, lambda *args: None, clickhouse_func)(results, genome_version) + backend_specific_call(lambda *args: None, lambda *args: None, _add_clickhouse_annotations)(results, genome_version) for result in results: result.update({k: v for k, v in result.pop('savedVariantJson').items() if k not in result}) @@ -469,14 +465,6 @@ def _add_clickhouse_annotations(results, genome_version): result.update(annotations_by_key[result['key']]) -# TODO revert? -def _safe_add_clickhouse_annotations(*args, **kwargs): - try: - _add_clickhouse_annotations(*args, **kwargs) - except Exception as e: - logger.error(f'Error loading saved variant annotations from clickhouse: {e}', None) - - def _format_functional_tags(tags): for tag in tags: name = tag.pop('functionalDataTag') From 445757c4395bd1fe914c2f675de2d974f11d17e1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 11 Sep 2025 16:24:18 -0400 Subject: [PATCH 49/89] udatetes --- matchmaker/views/matchmaker_api_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/matchmaker/views/matchmaker_api_tests.py b/matchmaker/views/matchmaker_api_tests.py index bd7e123046..bea9a7694c 100644 --- a/matchmaker/views/matchmaker_api_tests.py +++ b/matchmaker/views/matchmaker_api_tests.py @@ -12,7 +12,7 @@ update_mme_submission, delete_mme_submission, update_mme_result_status, send_mme_contact_email, \ get_mme_nodes, search_local_individual_mme_matches, finalize_mme_search, \ update_mme_contact_note, update_mme_project_contact -from seqr.views.utils.test_utils import AuthenticationTestCase +from seqr.views.utils.test_utils import AuthenticationTestCase, SAVED_VARIANT_FIELDS INDIVIDUAL_GUID = 'I000001_na19675' SUBMISSION_GUID = 'MS000001_na19675' @@ -230,6 +230,8 @@ def test_get_individual_mme_matches(self): set(response_json['savedVariantsByGuid'].keys()), {'SV0000001_2103343353_r0390_100', 'SV0059957_11562437_f019313_1', 'SV0059956_11560662_f019313_1'} ) + saved_variant_fields = {*SAVED_VARIANT_FIELDS, 'chrom', 'pos', 'transcripts', 'genomeVersion', 'noteGuids', 'functionalDataGuids', 'tagGuids'} + self.assertSetEqual(set(response_json['savedVariantsByGuid']['SV0000001_2103343353_r0390_100'].keys()), saved_variant_fields) self.assertDictEqual(response_json['mmeContactNotes'], {}) # users should see originating query for results if the have correct project permissions From 2879f9e3bd79a85bff5d2d8f7122300568e66e30 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 11 Sep 2025 16:53:02 -0400 Subject: [PATCH 50/89] codacy fix --- clickhouse_search/search.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index c0be75efe6..254c36c0e7 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -273,9 +273,7 @@ def get_transcripts_queryset(genome_version, keys): def get_transcripts_by_key(genome_version, keys): - return dict( - get_transcripts_queryset(genome_version, keys_with_transcripts).values_list('key', 'transcripts') - ) + return dict(get_transcripts_queryset(genome_version, keys).values_list('key', 'transcripts')) def format_clickhouse_results(results, genome_version, **kwargs): From 22049bbb577352a4e67cfa78c48d46fb9233f4c0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 11 Sep 2025 17:12:35 -0400 Subject: [PATCH 51/89] fix status tests --- seqr/views/status_tests.py | 44 ++++++++++++-------------------------- 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/seqr/views/status_tests.py b/seqr/views/status_tests.py index 8930b8a85d..a53a91d707 100644 --- a/seqr/views/status_tests.py +++ b/seqr/views/status_tests.py @@ -11,9 +11,19 @@ @mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') @mock.patch('seqr.views.status.redis.StrictRedis') @mock.patch('seqr.views.status.logger') -class StatusTest(object): +class ElasticsearchStatusTest(TestCase): + databases = '__all__' + + SEARCH_BACKEND_ERROR = 'No response from elasticsearch ping' + HAS_KIBANA = True + ES_HOSTNAME = 'testhost' + + def setUp(self): + patcher = mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', self.ES_HOSTNAME) + patcher.start() + self.addCleanup(patcher.stop) - def teardown(self): + def _post_teardown(self): for conn in connections.all(): conn.connection = None conn.ensure_connection() @@ -27,7 +37,6 @@ def test_status_error(self, mock_logger, mock_redis): conn.ensure_connection() conn.connection.close() mock_redis.return_value.ping.side_effect = HTTPError('Bad connection') - urllib3_responses.reset() urllib3_responses.add(urllib3_responses.HEAD, '/status', status=400) url = reverse(status_view) @@ -75,40 +84,15 @@ def test_status(self, mock_logger, mock_redis): mock_logger.error.assert_not_called() self._assert_expected_requests() - -class ElasticsearchStatusTest(TestCase, StatusTest): - databases = '__all__' - - SEARCH_BACKEND_ERROR = 'No response from elasticsearch ping' - HAS_KIBANA = True - - @mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') - def test_status(self, *args): - super(ElasticsearchStatusTest, self).test_status(*args) - def _assert_expected_requests(self): self.assertListEqual([call.request.url for call in urllib3_responses.calls], ['/', '/status', '/', '/status']) - def _post_teardown(self): - self.teardown() - super()._post_teardown() - - -class HailSearchStatusTest(TestCase, StatusTest): - databases = '__all__' +class HailSearchStatusTest(ElasticsearchStatusTest): SEARCH_BACKEND_ERROR = '400: Bad Request' HAS_KIBANA = False - - @mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', '') - @mock.patch('seqr.utils.search.hail_search_utils.HAIL_BACKEND_SERVICE_HOSTNAME', 'test-hail') - def test_status(self, *args): - super(HailSearchStatusTest, self).test_status(*args) + ES_HOSTNAME = '' def _assert_expected_requests(self): self.assertEqual(len(urllib3_responses.calls), 1) self.assertEqual(urllib3_responses.calls[0].request.url, '/status') - - def _post_teardown(self): - self.teardown() - super()._post_teardown() From 79558b9e13fbe19fb611c92babcd67dc52d20f69 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 12 Sep 2025 10:27:18 -0400 Subject: [PATCH 52/89] clean up --- .github/workflows/unit-tests.yml | 5 ----- .github/workflows/vlm-unit-tests.yaml | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index df87121123..56368fb8ef 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -69,11 +69,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - uses: actions/setup-java@v1 # TODO remove? - with: - java-version: "8" # The JDK version needed by hail - java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk - architecture: x64 # (x64 or x86) - defaults to x64 - name: Use pip cache uses: actions/cache@v4 with: diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index f8dcf4d23b..c5a570784e 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -21,7 +21,7 @@ on: jobs: vlm_clickhouse: runs-on: ubuntu-latest - container: hailgenetics/hail:0.2.128 # TODO different image? + container: hailgenetics/hail:0.2.128 services: clickhouse: From cc7c1f985c2e3e74eb6b55e4c8eb4e49943eba83 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Sep 2025 14:09:41 -0400 Subject: [PATCH 53/89] remove hail docker build file --- .cloudbuild/seqr-hail-search-docker.cloudbuild.yaml | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 .cloudbuild/seqr-hail-search-docker.cloudbuild.yaml diff --git a/.cloudbuild/seqr-hail-search-docker.cloudbuild.yaml b/.cloudbuild/seqr-hail-search-docker.cloudbuild.yaml deleted file mode 100644 index 91778b7b1c..0000000000 --- a/.cloudbuild/seqr-hail-search-docker.cloudbuild.yaml +++ /dev/null @@ -1,11 +0,0 @@ -steps: -- name: 'gcr.io/kaniko-project/executor:v1.3.0' - args: - - --destination=gcr.io/seqr-project/seqr-hail-search:${COMMIT_SHA} - - --destination=gcr.io/seqr-project/seqr-hail-search:${_CUSTOM_BRANCH_TAG} - - --destination=gcr.io/seqr-project/seqr-hail-search:latest - - --dockerfile=hail_search/deploy/Dockerfile - - --cache=true - - --cache-ttl=168h - -timeout: 1800s From 820ccf67609a99658f173bab326689227e116c88 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Sep 2025 14:24:42 -0400 Subject: [PATCH 54/89] clean up docker ignore --- .dockerignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index e1a4f868bb..e294067478 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,9 +5,7 @@ static/* !static/images/* deploy/* !deploy/docker/seqr/* -!deploy/docker/hail_search/* !deploy/docker/vlm/* -hail_search/fixtures/* .git .vscode .idea From 2013e1464714607c42a705bc1169ca43611e7478 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 12 Sep 2025 11:19:37 -0400 Subject: [PATCH 55/89] resolve merge --- matchmaker/views/matchmaker_api.py | 4 ++-- seqr/views/utils/project_context_utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/matchmaker/views/matchmaker_api.py b/matchmaker/views/matchmaker_api.py index a29a727748..f3a7dfffe4 100644 --- a/matchmaker/views/matchmaker_api.py +++ b/matchmaker/views/matchmaker_api.py @@ -54,11 +54,11 @@ def get_individual_mme_matches(request, submission_guid): SavedVariant.objects.filter(family=submission.individual.family), genome_version=project.genome_version, additional_values={ 'genomeVersion': Coalesce('saved_variant_json__genomeVersion', Value(project.genome_version), output_field=CharField()), 'transcripts': F('saved_variant_json__transcripts'), - }, additional_model_fields=backend_specific_call([], [], ['key', 'genotypes']), + }, additional_model_fields=backend_specific_call([], ['key', 'genotypes']), ) variants = response_json['savedVariantsByGuid'].values() - backend_specific_call(lambda *args: None, lambda *args: None, _add_clickhouse_transcripts)( + backend_specific_call(lambda *args: None, _add_clickhouse_transcripts)( variants, project.genome_version, request.user, ) diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index ddbf6b6fbc..82121c696f 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -126,7 +126,7 @@ def families_discovery_tags(families, genome_version, project=None): transcripts=F('saved_variant_json__transcripts'), mainTranscriptId=F('saved_variant_json__mainTranscriptId'), )} try: - backend_specific_call(lambda *args: None, lambda *args: None, _add_clickhouse_transcripts)(discovery_tags_by_key, genome_version) + backend_specific_call(lambda *args: None, _add_clickhouse_transcripts)(discovery_tags_by_key, genome_version) except Exception as e: logger.error(f'Error loading discovery genes from clickhouse: {e}', None) From 6d052b09d4ef1c0617582bcd5bde5fc0bc8401f0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 12 Sep 2025 11:35:43 -0400 Subject: [PATCH 56/89] move saved variant json reload to es specific module --- .../commands/reload_saved_variant_json.py | 3 +- seqr/utils/search/elasticsearch/es_utils.py | 37 ++++++++++++++++++- seqr/views/apis/saved_variant_api.py | 4 +- seqr/views/utils/variant_utils.py | 37 +------------------ 4 files changed, 42 insertions(+), 39 deletions(-) diff --git a/seqr/management/commands/reload_saved_variant_json.py b/seqr/management/commands/reload_saved_variant_json.py index 5766b328c3..cf59b16d3b 100644 --- a/seqr/management/commands/reload_saved_variant_json.py +++ b/seqr/management/commands/reload_saved_variant_json.py @@ -2,6 +2,7 @@ from django.core.management.base import BaseCommand, CommandError from django.db.models.query_utils import Q from seqr.models import Project +from seqr.utils.search.elasticsearch.es_utils import update_project_saved_variant_json from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.variant_utils import update_projects_saved_variant_json @@ -33,5 +34,5 @@ def handle(self, *args, **options): family_ids = [family_guid] if family_guid else None project_list = [(*project, family_ids) for project in projects.order_by('id').values_list('id', 'guid', 'name', 'genome_version')] - update_projects_saved_variant_json(project_list, user_email='manage_command') + update_projects_saved_variant_json(project_list, user_email='manage_command', update_function=update_project_saved_variant_json) logger.info("Done") diff --git a/seqr/utils/search/elasticsearch/es_utils.py b/seqr/utils/search/elasticsearch/es_utils.py index 94ffead964..051f2998cb 100644 --- a/seqr/utils/search/elasticsearch/es_utils.py +++ b/seqr/utils/search/elasticsearch/es_utils.py @@ -4,12 +4,13 @@ import elasticsearch_dsl from urllib3.connectionpool import connection_from_url -from seqr.models import Sample +from seqr.models import Sample, SavedVariant from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_set_json from seqr.utils.search.constants import VCF_FILE_EXTENSIONS, XPOS_SORT_KEY from seqr.utils.search.elasticsearch.es_gene_agg_search import EsGeneAggSearch from seqr.utils.search.elasticsearch.es_search import EsSearch, get_compound_het_page from seqr.views.utils.json_utils import _to_camel_case +from seqr.views.utils.variant_utils import get_variants_for_variant_ids, get_saved_variants from settings import ELASTICSEARCH_SERVICE_HOSTNAME, ELASTICSEARCH_SERVICE_PORT, ELASTICSEARCH_CREDENTIALS, \ ELASTICSEARCH_PROTOCOL, ES_SSL_CONTEXT, KIBANA_SERVER @@ -29,6 +30,8 @@ class InvalidIndexException(Exception): } ES_ERROR_LOG_EXCEPTIONS = {InvalidIndexException} +MAX_VARIANTS_FETCH = 1000 + def _get_transport_error_type(error): error_type = 'no detail' @@ -330,3 +333,35 @@ def process_es_previously_loaded_gene_aggs(previous_search_results): for family_guid in variants[0]['familyGuids']: gene_aggs[gene_id]['families'][family_guid] += len(variants) return gene_aggs + + +def update_project_saved_variant_json(project_id, genome_version, family_guids=None, dataset_type=None, user=None, user_email=None, **kwargs): + saved_variants = get_saved_variants(genome_version, project_id, family_guids, dataset_type).select_related('family') + + if not saved_variants: + return None + + families = set() + variant_ids = set() + saved_variants_map = {} + for v in saved_variants: + families.add(v.family) + variant_ids.add(v.variant_id) + saved_variants_map[(v.variant_id, v.family.guid)] = v + + variant_ids = sorted(variant_ids) + families = sorted(families, key=lambda f: f.guid) + variants_json = [] + for sub_var_ids in [variant_ids[i:i+MAX_VARIANTS_FETCH] for i in range(0, len(variant_ids), MAX_VARIANTS_FETCH)]: + variants_json += get_variants_for_variant_ids(families, sub_var_ids, user=user, user_email=user_email) + + updated_saved_variants = {} + for var in variants_json: + for family_guid in var['familyGuids']: + saved_variant = saved_variants_map.get((var['variantId'], family_guid)) + if saved_variant: + saved_variant.saved_variant_json = var + updated_saved_variants[saved_variant.guid] = saved_variant + SavedVariant.bulk_update_models(user, list(updated_saved_variants.values()), ['saved_variant_json']) + + return updated_saved_variants diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index e6ef867254..760b22ea15 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -8,6 +8,7 @@ from clickhouse_search.search import get_annotations_queryset, get_transcripts_queryset from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\ Family, GeneNote, Project +from seqr.utils.search.elasticsearch.es_utils import update_project_saved_variant_json from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json @@ -16,8 +17,7 @@ get_json_for_saved_variants_child_entities, get_json_for_gene_notes_by_gene_id, STRUCTURED_METADATA_TAG_TYPES from seqr.views.utils.permissions_utils import get_project_and_check_permissions, check_project_permissions, \ login_and_policies_required -from seqr.views.utils.variant_utils import update_project_saved_variant_json, reset_cached_search_results, \ - get_variants_response, parse_saved_variant_json +from seqr.views.utils.variant_utils import reset_cached_search_results, get_variants_response, parse_saved_variant_json logger = logging.getLogger(__name__) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index a4a4b59ba5..e416a3db42 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -30,19 +30,18 @@ logger = logging.getLogger(__name__) -MAX_VARIANTS_FETCH = 1000 DISCOVERY_CATEGORY = 'CMG Discovery Tags' OMIM_GENOME_VERSION = GENOME_VERSION_GRCh38 -def update_projects_saved_variant_json(projects, user_email, update_function=None, **kwargs): +def update_projects_saved_variant_json(projects, user_email, update_function, **kwargs): success = {} skipped = {} error = {} logger.info(f'Reloading saved variants in {len(projects)} projects') for project_id, project_guid, project_name, genome_version, family_guids in tqdm(projects, unit=' project'): try: - updated_saved_variants = (update_function or update_project_saved_variant_json)( + updated_saved_variants = update_function( project_id, genome_version, user_email=user_email, family_guids=family_guids, project_guid=project_guid, **kwargs) if updated_saved_variants is None: skipped[project_name] = True @@ -84,38 +83,6 @@ def get_saved_variants(genome_version, project_id=None, family_guids=None, datas return saved_variants -def update_project_saved_variant_json(project_id, genome_version, family_guids=None, dataset_type=None, user=None, user_email=None, **kwargs): - saved_variants = get_saved_variants(genome_version, project_id, family_guids, dataset_type).select_related('family') - - if not saved_variants: - return None - - families = set() - variant_ids = set() - saved_variants_map = {} - for v in saved_variants: - families.add(v.family) - variant_ids.add(v.variant_id) - saved_variants_map[(v.variant_id, v.family.guid)] = v - - variant_ids = sorted(variant_ids) - families = sorted(families, key=lambda f: f.guid) - variants_json = [] - for sub_var_ids in [variant_ids[i:i+MAX_VARIANTS_FETCH] for i in range(0, len(variant_ids), MAX_VARIANTS_FETCH)]: - variants_json += get_variants_for_variant_ids(families, sub_var_ids, user=user, user_email=user_email) - - updated_saved_variants = {} - for var in variants_json: - for family_guid in var['familyGuids']: - saved_variant = saved_variants_map.get((var['variantId'], family_guid)) - if saved_variant: - saved_variant.saved_variant_json = var - updated_saved_variants[saved_variant.guid] = saved_variant - SavedVariant.bulk_update_models(user, list(updated_saved_variants.values()), ['saved_variant_json']) - - return updated_saved_variants - - def saved_variants_dataset_type_filter(dataset_type): xpos_filter_key = 'xpos__gte' if dataset_type == Sample.DATASET_TYPE_MITO_CALLS else 'xpos__lt' dataset_filter = {xpos_filter_key: get_xpos('M', 1)} From dba183cca703d56cc3adcc38dd890d133fea338d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 12 Sep 2025 12:31:40 -0400 Subject: [PATCH 57/89] remove backend agnostic get variants by id --- seqr/utils/search/elasticsearch/es_utils.py | 23 +++++++------- seqr/utils/search/utils.py | 33 ++++----------------- seqr/views/utils/variant_utils.py | 16 ++++++---- 3 files changed, 29 insertions(+), 43 deletions(-) diff --git a/seqr/utils/search/elasticsearch/es_utils.py b/seqr/utils/search/elasticsearch/es_utils.py index 051f2998cb..a432104502 100644 --- a/seqr/utils/search/elasticsearch/es_utils.py +++ b/seqr/utils/search/elasticsearch/es_utils.py @@ -10,7 +10,7 @@ from seqr.utils.search.elasticsearch.es_gene_agg_search import EsGeneAggSearch from seqr.utils.search.elasticsearch.es_search import EsSearch, get_compound_het_page from seqr.views.utils.json_utils import _to_camel_case -from seqr.views.utils.variant_utils import get_variants_for_variant_ids, get_saved_variants +from seqr.views.utils.variant_utils import get_saved_variants from settings import ELASTICSEARCH_SERVICE_HOSTNAME, ELASTICSEARCH_SERVICE_PORT, ELASTICSEARCH_CREDENTIALS, \ ELASTICSEARCH_PROTOCOL, ES_SSL_CONTEXT, KIBANA_SERVER @@ -266,11 +266,12 @@ def _get_es_indices(client): return indices, seqr_index_projects -def get_es_variants_for_variant_ids(samples, genome_version, variants_by_id, user, return_all_queried_families=False, **kwargs): +def get_es_variants_for_variant_ids(samples, genome_version, variant_ids, user): + variant_ids = list(set(variant_ids)) variants = EsSearch( - samples, genome_version, user=user, return_all_queried_families=return_all_queried_families, sort=XPOS_SORT_KEY, - ).filter_by_variant_ids(list(variants_by_id.keys())) - return variants.search(num_results=len(variants_by_id)) + samples, genome_version, user=user, sort=XPOS_SORT_KEY, + ).filter_by_variant_ids(variant_ids) + return variants.search(num_results=len(variant_ids)) def get_es_variants(samples, search, user, previous_search_results, genome_version, sort=None, page=None, num_results=None, @@ -335,25 +336,25 @@ def process_es_previously_loaded_gene_aggs(previous_search_results): return gene_aggs -def update_project_saved_variant_json(project_id, genome_version, family_guids=None, dataset_type=None, user=None, user_email=None, **kwargs): - saved_variants = get_saved_variants(genome_version, project_id, family_guids, dataset_type).select_related('family') +def update_project_saved_variant_json(project_id, genome_version, family_guids=None, user=None, **kwargs): + saved_variants = get_saved_variants(genome_version, project_id, family_guids).select_related('family') if not saved_variants: return None - families = set() + family_ids = set() variant_ids = set() saved_variants_map = {} for v in saved_variants: - families.add(v.family) + family_ids.add(v.family_id) variant_ids.add(v.variant_id) saved_variants_map[(v.variant_id, v.family.guid)] = v variant_ids = sorted(variant_ids) - families = sorted(families, key=lambda f: f.guid) + samples = Sample.objects.filter(individual__family_id__in=family_ids, is_active=True) variants_json = [] for sub_var_ids in [variant_ids[i:i+MAX_VARIANTS_FETCH] for i in range(0, len(variant_ids), MAX_VARIANTS_FETCH)]: - variants_json += get_variants_for_variant_ids(families, sub_var_ids, user=user, user_email=user_email) + variants_json += get_es_variants_for_variant_ids(samples, genome_version, sub_var_ids, user) updated_saved_variants = {} for var in variants_json: diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 25bb6bad9e..32e65fc24e 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -136,46 +136,25 @@ def get_single_variant(family, variant_id, user=None): dataset_type = _variant_ids_dataset_type([parsed_variant_id]) samples = _get_families_search_data([family], dataset_type, sample_filter={'individual__family_id': family.id}) variant = backend_specific_call( - _process_ids_search(get_es_variants_for_variant_ids), + _get_es_variant_by_id, _get_clickhouse_variant_by_id, - )(parsed_variant_id, variant_id, samples, family.project.genome_version, dataset_type, user) + )(parsed_variant_id, variant_id, samples, family.project.genome_version, dataset_type=dataset_type, user=user) if not variant: raise InvalidSearchException('Variant {} not found'.format(variant_id)) return variant -def _process_ids_search(search_func): - def _search(parsed_variant_id, variant_id, samples, genome_version, dataset_type, user): - variants = search_func(samples, genome_version, {variant_id: parsed_variant_id}, user) - return variants[0] if variants else None - return _search +def _get_es_variant_by_id(parsed_variant_id, variant_id, samples, genome_version, user=None, **kwargs): + variants = get_es_variants_for_variant_ids(samples, genome_version, [variant_id], user) + return variants[0] if variants else None -def _get_clickhouse_variant_by_id(parsed_variant_id, variant_id, samples, genome_version, dataset_type, user): +def _get_clickhouse_variant_by_id(parsed_variant_id, variant_id, samples, genome_version, dataset_type=None, **kwargs): return get_clickhouse_variant_by_id( parsed_variant_id or variant_id, samples, genome_version, DATASET_TYPES_LOOKUP[dataset_type][0], ) -def get_variants_for_variant_ids(families, variant_ids, dataset_type=None, user=None, user_email=None, **kwargs): - parsed_variant_ids = {} - for variant_id in variant_ids: - parsed_variant_ids[variant_id] = parse_variant_id(variant_id) - - if dataset_type: - parsed_variant_ids = { - k: v for k, v in parsed_variant_ids.items() - if (dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and v) or - (dataset_type != Sample.DATASET_TYPE_VARIANT_CALLS and not v) - } - dataset_type = _variant_ids_dataset_type(parsed_variant_ids.values()) - - return backend_specific_call(get_es_variants_for_variant_ids, _raise_clickhouse_not_implemented)( - _get_families_search_data(families, dataset_type=dataset_type), _get_search_genome_version(families), - parsed_variant_ids, user, user_email=user_email, - ) - - def _variant_lookup(lookup_func, user, variant_id, dataset_type, genome_version=None, cache_key_suffix='', **kwargs): genome_version = genome_version or GENOME_VERSION_GRCh38 _validate_dataset_type_genome_version(dataset_type, genome_version) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index e416a3db42..a1663e3f65 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -15,7 +15,8 @@ from reference_data.models import TranscriptInfo, Omim, GENOME_VERSION_GRCh38 from seqr.models import SavedVariant, VariantSearchResults, Family, LocusList, LocusListInterval, LocusListGene, \ RnaSeqTpm, PhenotypePrioritization, Project, Sample, RnaSample, VariantTag, VariantTagType -from seqr.utils.search.utils import get_variants_for_variant_ids, backend_specific_call, parse_variant_id +from seqr.utils.search.elasticsearch.es_utils import get_es_variants_for_variant_ids +from seqr.utils.search.utils import backend_specific_call, parse_variant_id from seqr.utils.gene_utils import get_genes_for_variants from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.xpos_utils import get_xpos @@ -245,10 +246,11 @@ def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], user: U variant_families[variant_id].append(family_id) families_by_id = {f.id: f for f in Family.objects.filter(id__in=family_ids)} + samples = Sample.objects.filter(is_active=True, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) search_variants_by_id = { v['variantId']: v for v in backend_specific_call( - get_variants_for_variant_ids, _get_clickhouse_variants, - )(families=list(families_by_id.values()), variant_ids=list(variant_families.keys()), user=user, genome_version=genome_version) + _get_es_variants, _get_clickhouse_variants, + )(samples, families_by_id=families_by_id, variant_ids=list(variant_families.keys()), user=user, genome_version=genome_version) } new_variants = {} @@ -271,14 +273,18 @@ def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], user: U return new_variants -def _get_clickhouse_variants(families: list[Family], variant_ids: list[str], genome_version: str = None, **kwargs) -> list[dict]: +def _get_es_variants(samples: Sample.objects, families_by_id: dict[int, Family], *args, **kwargs): + return get_es_variants_for_variant_ids(samples.filter(individual__family_id__in=families_by_id.keys()), *args, **kwargs) + + +def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, Family], variant_ids: list[str], genome_version: str = None, **kwargs) -> list[dict]: variant_key_map = get_clickhouse_key_lookup(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variant_ids, reverse=True) + families = list(families_by_id.values()) prefetch_related_objects(families, 'project') families_by_project = defaultdict(list) for family in families: families_by_project[family.project.guid].append(family.guid) variants = [] - samples = Sample.objects.filter(is_active=True, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) for project_guid, family_guids in families_by_project.items(): genotype_keys = get_clickhouse_genotypes( project_guid, family_guids, genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variant_key_map.keys(), samples, From 261f0d0eb15df3a957e8183ff7243739babf5118 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 12 Sep 2025 12:33:20 -0400 Subject: [PATCH 58/89] remove unused exception --- seqr/utils/search/utils.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 32e65fc24e..6848094639 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -56,10 +56,6 @@ def _wrapped(*args, **kwargs): return _wrapped -def _raise_clickhouse_not_implemented(*args, **kwargs): - raise NotImplementedError('Clickhouse backend is not implemented for this function.') - - def backend_specific_call(es_func, clickhouse_func): if es_backend_enabled(): return es_func From 671329a8e2150a128a97585ecf32b2d8d5310843 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 12 Sep 2025 12:44:24 -0400 Subject: [PATCH 59/89] fix circular import --- seqr/utils/search/elasticsearch/es_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/utils/search/elasticsearch/es_utils.py b/seqr/utils/search/elasticsearch/es_utils.py index a432104502..fcae980d6a 100644 --- a/seqr/utils/search/elasticsearch/es_utils.py +++ b/seqr/utils/search/elasticsearch/es_utils.py @@ -10,7 +10,6 @@ from seqr.utils.search.elasticsearch.es_gene_agg_search import EsGeneAggSearch from seqr.utils.search.elasticsearch.es_search import EsSearch, get_compound_het_page from seqr.views.utils.json_utils import _to_camel_case -from seqr.views.utils.variant_utils import get_saved_variants from settings import ELASTICSEARCH_SERVICE_HOSTNAME, ELASTICSEARCH_SERVICE_PORT, ELASTICSEARCH_CREDENTIALS, \ ELASTICSEARCH_PROTOCOL, ES_SSL_CONTEXT, KIBANA_SERVER @@ -337,6 +336,7 @@ def process_es_previously_loaded_gene_aggs(previous_search_results): def update_project_saved_variant_json(project_id, genome_version, family_guids=None, user=None, **kwargs): + from seqr.views.utils.variant_utils import get_saved_variants saved_variants = get_saved_variants(genome_version, project_id, family_guids).select_related('family') if not saved_variants: From eafc23ae3c4a751221f16009acc816607a383902 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 12 Sep 2025 15:26:04 -0400 Subject: [PATCH 60/89] fix search --- seqr/utils/search/elasticsearch/es_utils.py | 2 +- .../search/elasticsearch/es_utils_tests.py | 7 +-- seqr/utils/search/search_utils_tests.py | 53 ++----------------- 3 files changed, 10 insertions(+), 52 deletions(-) diff --git a/seqr/utils/search/elasticsearch/es_utils.py b/seqr/utils/search/elasticsearch/es_utils.py index fcae980d6a..f6a564d104 100644 --- a/seqr/utils/search/elasticsearch/es_utils.py +++ b/seqr/utils/search/elasticsearch/es_utils.py @@ -266,7 +266,7 @@ def _get_es_indices(client): def get_es_variants_for_variant_ids(samples, genome_version, variant_ids, user): - variant_ids = list(set(variant_ids)) + variant_ids = sorted(set(variant_ids)) variants = EsSearch( samples, genome_version, user=user, sort=XPOS_SORT_KEY, ).filter_by_variant_ids(variant_ids) diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py index 787baed991..79015b80b9 100644 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ b/seqr/utils/search/elasticsearch/es_utils_tests.py @@ -13,7 +13,7 @@ from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.utils.search.utils import get_single_variant, query_variants, \ - get_variant_query_gene_counts, get_variants_for_variant_ids, InvalidSearchException + get_variant_query_gene_counts, get_es_variants_for_variant_ids, InvalidSearchException from seqr.utils.search.elasticsearch.es_search import _get_family_affected_status from seqr.utils.search.elasticsearch.es_utils import InvalidIndexException from seqr.utils.search.utils import _get_liftover @@ -1394,9 +1394,10 @@ def assertCachedResults(self, results_model, expected_results, sort='xpos'): @urllib3_responses.activate def test_get_es_variants_for_variant_ids(self): setup_responses() - get_variants_for_variant_ids(self.families, ['2-103343353-GAGA-G', '1-248367227-TC-T', 'prefix-938_DEL']) + samples = Sample.objects.filter(individual__family__in=self.families).exclude(dataset_type='MITO') + get_es_variants_for_variant_ids(samples, '37', ['2-103343353-GAGA-G', '1-248367227-TC-T', 'prefix-938_DEL'], None) self.assertExecutedSearch( - filters=[{'terms': {'variantId': ['2-103343353-GAGA-G', '1-248367227-TC-T', 'prefix-938_DEL']}}], + filters=[{'terms': {'variantId': ['1-248367227-TC-T', '2-103343353-GAGA-G', 'prefix-938_DEL']}}], size=6, index=','.join([INDEX_NAME, SV_INDEX_NAME]), ) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 80bf5b7b08..e0030c6ac3 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -8,7 +8,7 @@ GENE_COUNTS, VARIANT_LOOKUP_VARIANT, SV_VARIANT4, SV_VARIANT1 from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets -from seqr.utils.search.utils import get_single_variant, get_variants_for_variant_ids, get_variant_query_gene_counts, \ +from seqr.utils.search.utils import get_single_variant, get_variant_query_gene_counts, \ query_variants, variant_lookup, sv_variant_lookup, InvalidSearchException from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin, PARSED_VARIANTS, PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT, GENE_FIELDS @@ -127,48 +127,14 @@ def test_get_single_variant(self, mock_get_variants_for_ids): get_single_variant(family, '10-10334333-A-G') self.assertEqual(str(cm.exception), 'Variant 10-10334333-A-G not found') - def _assert_expected_get_single_variant_call(self, mock_get_variants_for_ids, parsed_variant_id, expected_samples, **kwargs): - if isinstance(parsed_variant_id, str): - variants_id_map = {parsed_variant_id: None} - else: - variants_id_map = {'-'.join([str(v) for v in parsed_variant_id]): parsed_variant_id} + def _assert_expected_get_single_variant_call(self, mock_get_variants_for_ids, variant_id, expected_samples, **kwargs): + if not isinstance(variant_id, str): + variant_id = '-'.join([str(v) for v in variant_id]) mock_get_variants_for_ids.assert_called_with( - mock.ANY, '37', variants_id_map, self.user, + mock.ANY, '37', [variant_id], self.user, ) self.assertSetEqual(set(mock_get_variants_for_ids.call_args.args[0]), expected_samples) - def test_get_variants_for_variant_ids(self, mock_get_variants_for_ids): - variant_ids = ['2-103343353-GAGA-G', '1-248367227-TC-T', 'prefix-938_DEL', 'MT-10195-C-A'] - get_variants_for_variant_ids(self.families, variant_ids, user=self.user) - mock_get_variants_for_ids.assert_called_with(mock.ANY, '37', { - '2-103343353-GAGA-G': ('2', 103343353, 'GAGA', 'G'), - '1-248367227-TC-T': ('1', 248367227, 'TC', 'T'), - 'MT-10195-C-A': ('M', 10195, 'C', 'A'), - 'prefix-938_DEL': None, - }, self.user, user_email=None) - self.assertSetEqual(set(mock_get_variants_for_ids.call_args.args[0]), set(self.search_samples)) - - get_variants_for_variant_ids( - self.families, variant_ids, user=self.user, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) - mock_get_variants_for_ids.assert_called_with(mock.ANY, '37', { - '2-103343353-GAGA-G': ('2', 103343353, 'GAGA', 'G'), - '1-248367227-TC-T': ('1', 248367227, 'TC', 'T'), - 'MT-10195-C-A': ('M', 10195, 'C', 'A'), - }, self.user, user_email=None) - skipped_samples = ['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733'] - expected_samples = {s for s in self.search_samples if s.guid not in skipped_samples} - self.assertSetEqual(set(mock_get_variants_for_ids.call_args.args[0]), expected_samples) - - get_variants_for_variant_ids( - self.families, variant_ids[:3], user=self.user, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) - mock_get_variants_for_ids.assert_called_with(mock.ANY, '37', { - '2-103343353-GAGA-G': ('2', 103343353, 'GAGA', 'G'), - '1-248367227-TC-T': ('1', 248367227, 'TC', 'T'), - }, self.user, user_email=None) - skipped_samples.append('S000149_hg00733') - expected_samples = {s for s in self.search_samples if s.guid not in skipped_samples} - self.assertSetEqual(set(mock_get_variants_for_ids.call_args.args[0]), expected_samples) - @mock.patch('seqr.utils.search.utils.MAX_FAMILY_COUNTS', {'WES': 2, 'WGS': 1}) @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) def _test_invalid_search_params(self, search_func): @@ -589,10 +555,6 @@ def test_get_single_variant(self, mock_get_variants_for_ids): mock_get_variants_for_ids.return_value = [PARSED_VARIANTS[0]] super(ElasticsearchSearchUtilsTests, self).test_get_single_variant(mock_get_variants_for_ids) - @mock.patch('seqr.utils.search.utils.get_es_variants_for_variant_ids') - def test_get_variants_for_variant_ids(self, mock_get_variants_for_ids): - super(ElasticsearchSearchUtilsTests, self).test_get_variants_for_variant_ids(mock_get_variants_for_ids) - @mock.patch('seqr.utils.search.utils.get_es_variants') def test_query_variants(self, mock_get_variants): super(ElasticsearchSearchUtilsTests, self).test_query_variants(mock_get_variants) @@ -695,11 +657,6 @@ def _assert_expected_get_single_variant_call(self, mock_call, variant_id, expect mock_call.assert_called_with(variant_id, mock.ANY, '37', dataset_type) self.assertSetEqual(set(mock_call.call_args.args[1]), expected_samples) - @mock.patch('seqr.utils.search.utils.get_clickhouse_variants') - def test_get_variants_for_variant_ids(self, mock_call): - with self.assertRaises(NotImplementedError): - super().test_get_variants_for_variant_ids(mock_call) - @mock.patch('seqr.utils.search.utils.get_clickhouse_variants') def test_query_variants(self, mock_call): super().test_query_variants(mock_call) From c3c72181b33947580461c40b44e7429b7aa74246 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 12 Sep 2025 15:52:25 -0400 Subject: [PATCH 61/89] restrict es only routes --- seqr/utils/search/utils.py | 25 ++++++++--------------- seqr/views/apis/data_manager_api.py | 14 ++++++++++--- seqr/views/apis/data_manager_api_tests.py | 22 ++++++++++++-------- 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 6848094639..b89062fd86 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -13,7 +13,7 @@ from seqr.utils.search.constants import XPOS_SORT_KEY, PRIORITIZED_GENE_SORT, RECESSIVE, COMPOUND_HET, \ MAX_NO_LOCATION_COMP_HET_FAMILIES, SV_ANNOTATION_TYPES, ALL_DATA_TYPES, MAX_EXPORT_VARIANTS, X_LINKED_RECESSIVE, \ MAX_VARIANTS -from seqr.utils.search.elasticsearch.es_utils import ping_elasticsearch, delete_es_index, get_elasticsearch_status, \ +from seqr.utils.search.elasticsearch.es_utils import ping_elasticsearch, \ get_es_variants, get_es_variants_for_variant_ids, process_es_previously_loaded_results, process_es_previously_loaded_gene_aggs, \ es_backend_enabled, ping_kibana, ES_EXCEPTION_ERROR_MAP, ES_EXCEPTION_MESSAGE_MAP, ES_ERROR_LOG_EXCEPTIONS from seqr.utils.gene_utils import parse_locus_list_items @@ -56,6 +56,14 @@ def _wrapped(*args, **kwargs): return _wrapped +def es_only(func): + def _wrapped(*args, **kwargs): + if not es_backend_enabled(): + raise ValueError(f'{func.__name__} is disabled without the elasticsearch backend') + return func(*args, **kwargs) + return _wrapped + + def backend_specific_call(es_func, clickhouse_func): if es_backend_enabled(): return es_func @@ -72,10 +80,6 @@ def ping_search_backend_admin(): backend_specific_call(ping_kibana, lambda: True)() -def get_search_backend_status(): - return backend_specific_call(get_elasticsearch_status, _raise_search_error('Elasticsearch is disabled'))() - - def _get_filtered_search_samples(search_filter, active_only=True): samples = Sample.objects.filter(**search_filter) if active_only: @@ -116,17 +120,6 @@ def _get_search_genome_version(families): return next(iter(project_versions.keys())) -def delete_search_backend_data(data_id): - active_samples = Sample.objects.filter(is_active=True, elasticsearch_index=data_id) - if active_samples: - projects = set(active_samples.values_list('individual__family__project__name', flat=True)) - raise InvalidSearchException(f'"{data_id}" is still used by: {", ".join(projects)}') - - return backend_specific_call( - delete_es_index, _raise_search_error('Deleting indices is disabled without the elasticsearch backend'), - )(data_id) - - def get_single_variant(family, variant_id, user=None): parsed_variant_id = parse_variant_id(variant_id) dataset_type = _variant_ids_dataset_type([parsed_variant_id]) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index b7bd5e109e..58580320ae 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -16,7 +16,8 @@ from clickhouse_search.search import delete_clickhouse_project, delete_clickhouse_family from seqr.utils.communication_utils import send_project_notification from seqr.utils.search.add_data_utils import prepare_data_loading_request, get_loading_samples_validator -from seqr.utils.search.utils import get_search_backend_status, delete_search_backend_data, backend_specific_call +from seqr.utils.search.elasticsearch.es_utils import get_elasticsearch_status, delete_es_index +from seqr.utils.search.utils import backend_specific_call, es_only, InvalidSearchException from seqr.utils.file_utils import file_iter, does_file_exist from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException @@ -43,14 +44,21 @@ @data_manager_required +@es_only def elasticsearch_status(request): - return create_json_response(get_search_backend_status()) + return create_json_response(get_elasticsearch_status()) @data_manager_required +@es_only def delete_index(request): index = json.loads(request.body)['index'] - updated_indices = delete_search_backend_data(index) + active_samples = Sample.objects.filter(is_active=True, elasticsearch_index=index) + if active_samples: + projects = set(active_samples.values_list('individual__family__project__name', flat=True)) + raise InvalidSearchException(f'"{index}" is still used by: {", ".join(projects)}') + + updated_indices = delete_es_index(index) return create_json_response({'indices': updated_indices}) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index d9dedc52e5..f6db801145 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -506,10 +506,7 @@ def test_delete_index(self): self.check_data_manager_login(url) response = self.client.post(url, content_type='application/json', data=json.dumps({'index': 'test_index'})) - self.assertEqual(response.status_code, 400) - self.assertEqual( - response.json()['error'], '"test_index" is still used by: 1kg project n\xe5me with uni\xe7\xf8de') - self.assertEqual(len(urllib3_responses.calls), 0) + self._assert_expected_delete_active_index_response(response) urllib3_responses.add_json( '/_cat/indices?format=json&h=index,docs.count,store.size,creation.date.string', ES_CAT_INDICES) @@ -520,6 +517,12 @@ def test_delete_index(self): response = self.client.post(url, content_type='application/json', data=json.dumps({'index': 'unused_index'})) self._assert_expected_delete_index_response(response) + def _assert_expected_delete_active_index_response(self, response): + self.assertEqual(response.status_code, 400) + self.assertEqual( + response.json()['error'], '"test_index" is still used by: 1kg project n\xe5me with uni\xe7\xf8de') + self.assertEqual(len(urllib3_responses.calls), 0) + def _assert_expected_delete_index_response(self, response): self.assertEqual(response.status_code, 200) response_json = response.json() @@ -1806,12 +1809,15 @@ def _additional_expected_loading_subprocess_calls(file_path): return [f'gsutil mv tmp/temp_uploads/{file_path} gs://seqr-scratch-temp/{file_path}'] def _assert_expected_es_status(self, response): - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['error'], 'Elasticsearch is disabled') + self.assertEqual(response.status_code, 500) + self.assertEqual(response.json()['error'], 'elasticsearch_status is disabled without the elasticsearch backend') def _assert_expected_delete_index_response(self, response): - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['error'], 'Deleting indices is disabled without the elasticsearch backend') + self.assertEqual(response.status_code, 500) + self.assertEqual(response.json()['error'], 'delete_index is disabled without the elasticsearch backend') + + def _assert_expected_delete_active_index_response(self, response): + self._assert_expected_delete_index_response(response) def _assert_expected_get_projects_requests(self): pdo_filter = "OR(SEARCH('Methods (Loading)',ARRAYJOIN(PDOStatus,';')),SEARCH('On hold for phenotips, but ready to load',ARRAYJOIN(PDOStatus,';')))" From f13205cb7769c46754c1039604f2023fbc6eb0d7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 12 Sep 2025 15:57:37 -0400 Subject: [PATCH 62/89] restrict clickhouse only routes --- seqr/utils/search/utils.py | 8 ++++++++ seqr/views/apis/data_manager_api.py | 8 ++++---- seqr/views/apis/data_manager_api_tests.py | 10 ++++------ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index b89062fd86..e03416637d 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -64,6 +64,14 @@ def _wrapped(*args, **kwargs): return _wrapped +def clickhouse_only(func): + def _wrapped(*args, **kwargs): + if es_backend_enabled(): + raise ValueError(f'{func.__name__} is disabled without the clickhouse backend') + return func(*args, **kwargs) + return _wrapped + + def backend_specific_call(es_func, clickhouse_func): if es_backend_enabled(): return es_func diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 58580320ae..00c3362e84 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -17,7 +17,7 @@ from seqr.utils.communication_utils import send_project_notification from seqr.utils.search.add_data_utils import prepare_data_loading_request, get_loading_samples_validator from seqr.utils.search.elasticsearch.es_utils import get_elasticsearch_status, delete_es_index -from seqr.utils.search.utils import backend_specific_call, es_only, InvalidSearchException +from seqr.utils.search.utils import clickhouse_only, es_only, InvalidSearchException from seqr.utils.file_utils import file_iter, does_file_exist from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException @@ -432,6 +432,7 @@ def fetch_missing_vcf_samples(missing_vcf_samples): @data_manager_required +@clickhouse_only def trigger_delete_project(request): request_json = json.loads(request.body) project_guid = request_json.pop('project') @@ -440,6 +441,7 @@ def trigger_delete_project(request): @data_manager_required +@clickhouse_only def trigger_delete_family(request): request_json = json.loads(request.body) family_guid = request_json.pop('family') @@ -458,9 +460,7 @@ def _trigger_data_update(clickhouse_func, request_json, project, *args): ).values_list('sample_type', flat=True).distinct() if dataset_type == Sample.DATASET_TYPE_SV_CALLS else [None] info = [] for sample_type in sample_types: - info.append(backend_specific_call( - _raise_backend_not_implemented, clickhouse_func, - )(project, *args, dataset_type=dataset_type, sample_type=sample_type)) + info.append(clickhouse_func(project, *args, dataset_type=dataset_type, sample_type=sample_type)) return create_json_response({'info': info}) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index f6db801145..035be59d6d 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1570,14 +1570,12 @@ def test_trigger_delete_family(self): self._assert_expected_delete_family(response) def _assert_expected_delete_project(self, response): - self._assert_expected_search_data_update(response) + self.assertEqual(response.status_code, 500) + self.assertDictEqual(response.json(), {'error': 'trigger_delete_project is disabled without the clickhouse backend'}) def _assert_expected_delete_family(self, response): - self._assert_expected_search_data_update(response) - - def _assert_expected_search_data_update(self, response): - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['This functionality is not available in the current search backend'], 'warnings': None}) + self.assertEqual(response.status_code, 500) + self.assertDictEqual(response.json(), {'error': 'trigger_delete_family is disabled without the clickhouse backend'}) class LocalDataManagerAPITest(AuthenticationTestCase, DataManagerAPITest): From 3fd0c38aa3deadbdc319782728ce39ab940901ee Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 12 Sep 2025 16:21:38 -0400 Subject: [PATCH 63/89] restrict update saved variants to es --- seqr/views/apis/data_manager_api.py | 4 --- seqr/views/apis/saved_variant_api.py | 8 ++--- seqr/views/apis/saved_variant_api_tests.py | 34 +++++++++++----------- 3 files changed, 19 insertions(+), 27 deletions(-) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 00c3362e84..7dfde66a4e 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -449,10 +449,6 @@ def trigger_delete_family(request): return _trigger_data_update(delete_clickhouse_family, request_json, project, family_guid) -def _raise_backend_not_implemented(*args, **kwargs): - raise ErrorsWarningsException(['This functionality is not available in the current search backend']) - - def _trigger_data_update(clickhouse_func, request_json, project, *args): dataset_type = request_json.get('datasetType') sample_types = Sample.objects.filter( diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 760b22ea15..8a75fb697b 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -9,7 +9,7 @@ from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\ Family, GeneNote, Project from seqr.utils.search.elasticsearch.es_utils import update_project_saved_variant_json -from seqr.utils.search.utils import backend_specific_call +from seqr.utils.search.utils import backend_specific_call, es_only from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json from seqr.views.utils.json_utils import create_json_response @@ -319,8 +319,8 @@ def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=Vari @login_and_policies_required +@es_only def update_saved_variant_json(request, project_guid): - backend_specific_call(lambda: True, _backend_error)() project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) reset_cached_search_results(project) try: @@ -332,10 +332,6 @@ def update_saved_variant_json(request, project_guid): return create_json_response({variant_guid: None for variant_guid in updated_saved_variant_guids or []}) -def _backend_error(*args, **kwargs): - raise ValueError('Endpoint is disabled for current search backend') - - @login_and_policies_required def update_variant_main_transcript(request, variant_guid, transcript_id): saved_variant = SavedVariant.objects.get(guid=variant_guid) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 98b235f636..9c13be3624 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -4,7 +4,7 @@ from django.urls.base import reverse -from seqr.models import SavedVariant, VariantNote, VariantTag, VariantFunctionalData, Family, Project +from seqr.models import SavedVariant, VariantNote, VariantTag, VariantFunctionalData, Sample, Project from seqr.views.apis.saved_variant_api import saved_variant_data, create_variant_note_handler, create_saved_variant_handler, \ update_variant_note_handler, delete_variant_note_handler, update_variant_tags_handler, update_saved_variant_json, \ update_variant_main_transcript, update_variant_functional_data_handler, update_variant_acmg_classification_handler @@ -953,20 +953,19 @@ def test_update_compound_hets_variant_functional_data(self): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': 'Unable to find the following variant(s): not_variant'}) - @mock.patch('seqr.views.utils.variant_utils.MAX_VARIANTS_FETCH', 2) - @mock.patch('seqr.utils.search.utils.es_backend_enabled') + @mock.patch('seqr.utils.search.elasticsearch.es_utils.MAX_VARIANTS_FETCH', 2) @mock.patch('seqr.views.apis.saved_variant_api.logger') - @mock.patch('seqr.views.utils.variant_utils.get_variants_for_variant_ids') - def test_update_saved_variant_json(self, mock_get_variants, mock_logger, mock_es_enabled): - mock_es_enabled.return_value = True - mock_get_variants.side_effect = lambda families, variant_ids, **kwargs: \ - [{'variantId': variant_id, 'familyGuids': [family.guid for family in families]} + @mock.patch('seqr.utils.search.elasticsearch.es_utils.get_es_variants_for_variant_ids') + def test_update_saved_variant_json(self, mock_get_variants, mock_logger): + mock_get_variants.side_effect = lambda samples, genome_version, variant_ids, user: \ + [{'variantId': variant_id, 'familyGuids': sorted({sample.individual.family.guid for sample in samples})} for variant_id in variant_ids] url = reverse(update_saved_variant_json, args=['R0001_1kg']) self.check_manager_login(url) response = self.client.post(url) + self.assertEqual(response.status_code, 200) self.assertDictEqual( @@ -975,11 +974,13 @@ def test_update_saved_variant_json(self, mock_get_variants, mock_logger, mock_es 'SV0059956_11560662_f019313_1': None} ) - families = [Family.objects.get(guid='F000001_1'), Family.objects.get(guid='F000002_2')] mock_get_variants.assert_has_calls([ - mock.call(families, ['1-248367227-TC-T', '1-46859832-G-A'], user=self.manager_user, user_email=None), - mock.call(families, ['21-3343353-GAGA-G'], user=self.manager_user, user_email=None), + mock.call(mock.ANY, '37', ['1-248367227-TC-T', '1-46859832-G-A'], self.manager_user), + mock.call(mock.ANY, '37', ['21-3343353-GAGA-G'], self.manager_user), ]) + samples = set(Sample.objects.filter(individual__family_id__in=[1, 2], is_active=True).values_list('id', flat=True)) + self.assertSetEqual(set(mock_get_variants.call_args_list[0].args[0].values_list('id', flat=True)), samples) + self.assertSetEqual(set(mock_get_variants.call_args_list[1].args[0].values_list('id', flat=True)), samples) mock_logger.error.assert_not_called() # Test handles update error @@ -988,10 +989,6 @@ def test_update_saved_variant_json(self, mock_get_variants, mock_logger, mock_es self.assertEqual(response.status_code, 200) mock_logger.error.assert_called_with('Unable to reset saved variant json for R0001_1kg: Unable to fetch variants') - mock_es_enabled.return_value = False - response = self.client.post(url) - self.assertEqual(response.status_code, 500) - def test_update_variant_main_transcript(self): transcript_id = 'ENST00000438943' update_main_transcript_url = reverse(update_variant_main_transcript, args=[VARIANT_GUID, transcript_id]) @@ -1106,8 +1103,11 @@ def test_update_compound_hets_variant_functional_data(self): assert_no_list_ws_has_al(self, 3) def test_update_saved_variant_json(self, *args): - super(AnvilSavedVariantAPITest, self).test_update_saved_variant_json(*args) - assert_no_list_ws_has_al(self, 3) + url = reverse(update_saved_variant_json, args=['R0001_1kg']) + self.check_require_login(url) + response = self.client.post(url) + self.assertEqual(response.status_code, 500) + self.assertDictEqual(response.json(), {'error': 'update_saved_variant_json is disabled without the elasticsearch backend'}) def test_update_variant_main_transcript(self): super(AnvilSavedVariantAPITest, self).test_update_variant_main_transcript() From 2bf5350073919465f3c6d8391dae3021c9460921 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 12 Sep 2025 16:42:32 -0400 Subject: [PATCH 64/89] fix test --- .../commands/reload_saved_variant_json.py | 8 ++--- .../tests/reload_saved_variant_json_tests.py | 33 ++++++++++--------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/seqr/management/commands/reload_saved_variant_json.py b/seqr/management/commands/reload_saved_variant_json.py index cf59b16d3b..6d64f991e7 100644 --- a/seqr/management/commands/reload_saved_variant_json.py +++ b/seqr/management/commands/reload_saved_variant_json.py @@ -3,16 +3,12 @@ from django.db.models.query_utils import Q from seqr.models import Project from seqr.utils.search.elasticsearch.es_utils import update_project_saved_variant_json -from seqr.utils.search.utils import backend_specific_call +from seqr.utils.search.utils import es_only from seqr.views.utils.variant_utils import update_projects_saved_variant_json logger = logging.getLogger(__name__) -def _clickhouse_error(): - raise CommandError('Reloading variants is not supported in clickhouse') - - class Command(BaseCommand): help = 'Transfer projects to the new seqr schema' @@ -20,8 +16,8 @@ def add_arguments(self, parser): parser.add_argument('projects', nargs="*", help='Project(s) to transfer. If not specified, defaults to all projects.') parser.add_argument('--family-guid', help='optional family to reload variants for') + @es_only def handle(self, *args, **options): - backend_specific_call(lambda: True, _clickhouse_error)() projects_to_process = options['projects'] family_guid = options['family_guid'] diff --git a/seqr/management/tests/reload_saved_variant_json_tests.py b/seqr/management/tests/reload_saved_variant_json_tests.py index 1c885dbd70..ae5983fa0f 100644 --- a/seqr/management/tests/reload_saved_variant_json_tests.py +++ b/seqr/management/tests/reload_saved_variant_json_tests.py @@ -3,7 +3,6 @@ from django.core.management import call_command, CommandError from django.test import TestCase -from seqr.models import Family PROJECT_NAME = '1kg project n\u00e5me with uni\u00e7\u00f8de' PROJECT_GUID = 'R0001_1kg' @@ -15,10 +14,10 @@ class ReloadSavedVariantJsonTest(TestCase): fixtures = ['users', '1kg_project'] @mock.patch('seqr.views.utils.variant_utils.logger') - @mock.patch('seqr.views.utils.variant_utils.get_variants_for_variant_ids') + @mock.patch('seqr.utils.search.elasticsearch.es_utils.get_es_variants_for_variant_ids') def test_with_param_command(self, mock_get_variants, mock_logger): - mock_get_variants.side_effect = lambda families, variant_ids, **kwargs: \ - [{'variantId': variant_id, 'familyGuids': [family.guid for family in families]} + mock_get_variants.side_effect = lambda samples, genome_version, variant_ids, user: \ + [{'variantId': variant_id, 'familyGuids': sorted({sample.individual.family.guid for sample in samples}) if samples else ['F000012_12']} for variant_id in variant_ids] # Test with a specific project and a family id. @@ -26,9 +25,8 @@ def test_with_param_command(self, mock_get_variants, mock_logger): PROJECT_NAME, '--family-guid={}'.format(FAMILY_GUID)) - family_1 = Family.objects.get(id=1) - mock_get_variants.assert_called_with( - [family_1], ['1-46859832-G-A','21-3343353-GAGA-G'], user=None, user_email='manage_command') + mock_get_variants.assert_called_with(mock.ANY, '37', ['1-46859832-G-A','21-3343353-GAGA-G'], None) + self.assertSetEqual(set(mock_get_variants.call_args_list[0].args[0].values_list('id', flat=True)), {129, 130}) logger_info_calls = [ mock.call('Updated 2 variants in 1 families for project 1kg project n\xe5me with uni\xe7\xf8de'), @@ -43,14 +41,18 @@ def test_with_param_command(self, mock_get_variants, mock_logger): call_command('reload_saved_variant_json') self.assertEqual(mock_get_variants.call_count, 3) - family_2 = Family.objects.get(id=2) mock_get_variants.assert_has_calls([ mock.call( - [family_1, family_2], ['1-248367227-TC-T', '1-46859832-G-A', '21-3343353-GAGA-G'], user=None, user_email='manage_command', + mock.ANY, '37', ['1-248367227-TC-T', '1-46859832-G-A', '21-3343353-GAGA-G'], None, ), - mock.call([Family.objects.get(id=12)], ['1-248367227-TC-T', 'prefix_19107_DEL'], user=None, user_email='manage_command'), - mock.call([Family.objects.get(id=14)], ['1-248367227-TC-T'], user=None, user_email='manage_command') - ], any_order=True) + mock.call(mock.ANY, '37', ['1-248367227-TC-T', 'prefix_19107_DEL'], None), + mock.call(mock.ANY, '38', ['1-248367227-TC-T'], None) + ]) + self.assertSetEqual(set(mock_get_variants.call_args_list[0].args[0].values_list('id', flat=True)), { + 129, 130, 132, 133, 134, 145, 146, 148, 149, + }) + self.assertSetEqual(set(mock_get_variants.call_args_list[1].args[0].values_list('id', flat=True)), set()) + self.assertSetEqual(set(mock_get_variants.call_args_list[2].args[0].values_list('id', flat=True)), {147, 173, 174}) logger_info_calls = [ mock.call('Reloading saved variants in 4 projects'), @@ -73,7 +75,8 @@ def test_with_param_command(self, mock_get_variants, mock_logger): PROJECT_GUID, '--family-guid={}'.format(FAMILY_GUID)) - mock_get_variants.assert_called_with([family_1], ['1-46859832-G-A', '21-3343353-GAGA-G'], user=None, user_email='manage_command') + mock_get_variants.assert_called_with(mock.ANY, '37', ['1-46859832-G-A', '21-3343353-GAGA-G'], None) + self.assertSetEqual(set(mock_get_variants.call_args_list[0].args[0].values_list('id', flat=True)), {129, 130}) logger_info_calls = [ mock.call('Reload Summary: '), @@ -91,8 +94,8 @@ class ClickhouseReloadSavedVariantJsonTest(TestCase): fixtures = ['users', '1kg_project'] def test_command(self): - with self.assertRaises(CommandError) as ce: + with self.assertRaises(ValueError) as ce: call_command('reload_saved_variant_json', PROJECT_NAME) - self.assertEqual(str(ce.exception), 'Reloading variants is not supported in clickhouse') + self.assertEqual(str(ce.exception), 'handle is disabled without the elasticsearch backend') From f867b82c42e11e7047fee2ffe594c32027028fa8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 12 Sep 2025 16:49:32 -0400 Subject: [PATCH 65/89] fix test --- seqr/views/apis/summary_data_api_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index a917d78829..7df3cad33e 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -839,7 +839,7 @@ def _test_metadata_airtable_responses(self, include_airtable_url, expected_indiv self.assertEqual(response.status_code, 200) self._has_expected_metadata_response(response, expected_individuals) - @mock.patch('seqr.views.utils.variant_utils.get_variants_for_variant_ids', lambda *args, **kwargs: PARSED_VARIANTS) + @mock.patch('seqr.views.utils.variant_utils.get_es_variants_for_variant_ids', lambda *args, **kwargs: PARSED_VARIANTS) def test_bulk_update_family_external_analysis(self, *args, **kwargs): super().test_bulk_update_family_external_analysis(*args, **kwargs) From c147e10a4a353b999a8622c10416405aa0696192 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 12 Sep 2025 17:11:50 -0400 Subject: [PATCH 66/89] first pass move lookup to clickhosue --- clickhouse_search/search.py | 62 ++++++++++++++++++++++-- seqr/utils/search/utils.py | 69 +-------------------------- seqr/views/apis/variant_search_api.py | 13 +++-- 3 files changed, 69 insertions(+), 75 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 8916ed2054..f495ca6147 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -1,5 +1,6 @@ from clickhouse_backend.models import ArrayField, StringField from collections import defaultdict +from datetime import timedelta from django.contrib.postgres.aggregates import ArrayAgg from django.core.exceptions import ObjectDoesNotExist from django.db import connections @@ -12,13 +13,13 @@ ArrayDistinct, ArrayMap from clickhouse_search.models import ENTRY_CLASS_MAP, ANNOTATIONS_CLASS_MAP, TRANSCRIPTS_CLASS_MAP, KEY_LOOKUP_CLASS_MAP, \ BaseClinvar, BaseAnnotationsMitoSnvIndel, BaseAnnotationsGRCh37SnvIndel, BaseAnnotationsSvGcnv -from reference_data.models import GeneConstraint, Omim, GENOME_VERSION_LOOKUP +from reference_data.models import GeneConstraint, Omim, GENOME_VERSION_LOOKUP, GENOME_VERSION_GRCh37 from seqr.models import Sample, PhenotypePrioritization, Individual from seqr.utils.logging_utils import SeqrLogger +from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_set_json from seqr.utils.search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \ PRIORITIZED_GENE_SORT, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS, RECESSIVE from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets -from settings import CLICKHOUSE_SERVICE_HOSTNAME logger = SeqrLogger(__name__) @@ -552,7 +553,7 @@ def _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples): variant = format_clickhouse_results([variant], genome_version)[0] return variant -def clickhouse_variant_lookup(user, variant_id, data_type, genome_version=None, samples=None, **kwargs): +def clickhouse_variant_lookup(user, variant_id, data_type, genome_version, samples=None, **kwargs): logger.info(f'Looking up variant {variant_id} with data type {data_type}', user) variant = _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples) @@ -607,6 +608,61 @@ def get_clickhouse_variant_by_id(variant_id, samples, genome_version, dataset_ty return None +def _variant_lookup(lookup_func, user, variant_id, genome_version, dataset_type, cache_key_suffix='', **kwargs): + _validate_dataset_type_genome_version(dataset_type, genome_version) + cache_key = f'variant_lookup_results__{variant_id}__{genome_version}__{cache_key_suffix}' + variant = safe_redis_get_json(cache_key) + if variant: + return variant + + variant = lookup_func(user, variant_id, dataset_type, genome_version, **kwargs) + safe_redis_set_json(cache_key, variant, expire=timedelta(weeks=2)) + return variant + + +def _validate_dataset_type_genome_version(dataset_type, genome_version): + if genome_version == GENOME_VERSION_GRCh37 and dataset_type != Sample.DATASET_TYPE_VARIANT_CALLS: + from seqr.utils.search.utils import InvalidSearchException + raise InvalidSearchException(f'{dataset_type} variants are not available for GRCh37') + + +def variant_lookup(user, parsed_variant_id, genome_version, **kwargs): + dataset_type = Sample.DATASET_TYPE_MITO_CALLS if parsed_variant_id[0].replace('chr', '').startswith('M') else Sample.DATASET_TYPE_VARIANT_CALLS + return _variant_lookup(clickhouse_variant_lookup, user, parsed_variant_id, genome_version, dataset_type, **kwargs) + + +def sv_variant_lookup(user, variant_id, genome_version, families, sample_type, **kwargs): + samples = _get_families_search_data(families, dataset_type=Sample.DATASET_TYPE_SV_CALLS) # TODO + + return _variant_lookup( + _sv_variant_lookup, user, variant_id, genome_version, dataset_type=Sample.DATASET_TYPE_SV_CALLS, + **kwargs, samples=samples, sample_type=sample_type, cache_key_suffix=user, + ) + + +def _sv_variant_lookup(user, variant_id, dataset_type, genome_version, samples, sample_type, **kwargs): + data_type = f'{dataset_type}_{sample_type}' + + lookup_samples = samples.filter(sample_type=sample_type) + variant = clickhouse_variant_lookup(user, variant_id, data_type, samples=lookup_samples, genome_version=genome_version, **kwargs) + variants = [variant] + + if variant['svType'] in {'DEL', 'DUP'}: + samples = samples.exclude(sample_type=sample_type) + search = { + 'parsed_locus': { + 'padded_interval': {'chrom': variant['chrom'], 'start': variant['pos'], 'end': variant['end'], 'padding': 0.2}, + }, + 'annotations': {'structural': [variant['svType'], f"gCNV_{variant['svType']}"]}, + **kwargs, + } + results = {} + _execute_search(samples, search, user, previous_search_results=results, genome_version=genome_version) # TODO + variants += results['all_results'] + + return variants + + def get_clickhouse_genotypes(project_guid, family_guids, genome_version, dataset_type, keys, samples): sample_data = _get_sample_data(samples.filter(individual__family__guid__in=family_guids))[dataset_type] entries = ENTRY_CLASS_MAP[genome_version][dataset_type].objects.filter( diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index e03416637d..f6411e05a7 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -5,7 +5,7 @@ from pyliftover.liftover import LiftOver from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, \ - get_clickhouse_cache_results, clickhouse_variant_lookup, get_clickhouse_variant_by_id + get_clickhouse_cache_results, get_clickhouse_variant_by_id from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 from seqr.models import Sample, Individual, Project from seqr.utils.logging_utils import SeqrLogger @@ -50,12 +50,6 @@ class InvalidSearchException(Exception): DATASET_TYPES_LOOKUP[DATASET_TYPE_NO_MITO] = [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_SV_CALLS] -def _raise_search_error(error): - def _wrapped(*args, **kwargs): - raise InvalidSearchException(error) - return _wrapped - - def es_only(func): def _wrapped(*args, **kwargs): if not es_backend_enabled(): @@ -152,67 +146,6 @@ def _get_clickhouse_variant_by_id(parsed_variant_id, variant_id, samples, genome ) -def _variant_lookup(lookup_func, user, variant_id, dataset_type, genome_version=None, cache_key_suffix='', **kwargs): - genome_version = genome_version or GENOME_VERSION_GRCh38 - _validate_dataset_type_genome_version(dataset_type, genome_version) - cache_key = f'variant_lookup_results__{variant_id}__{genome_version}__{cache_key_suffix}' - variant = safe_redis_get_json(cache_key) - if variant: - return variant - - variant = lookup_func(user, variant_id, dataset_type, genome_version=genome_version, **kwargs) - safe_redis_set_json(cache_key, variant, expire=timedelta(weeks=2)) - return variant - - -def _validate_dataset_type_genome_version(dataset_type, genome_version): - if genome_version == GENOME_VERSION_GRCh37 and dataset_type != Sample.DATASET_TYPE_VARIANT_CALLS: - raise InvalidSearchException(f'{dataset_type} variants are not available for GRCh37') - - -def variant_lookup(user, parsed_variant_id, **kwargs): - dataset_type = DATASET_TYPES_LOOKUP[_variant_ids_dataset_type([parsed_variant_id])][0] - lookup_func = backend_specific_call(_raise_search_error('Lookup is disabled'), clickhouse_variant_lookup) - return _variant_lookup(lookup_func, user, parsed_variant_id, **kwargs, dataset_type=dataset_type) - - -def sv_variant_lookup(user, variant_id, families, **kwargs): - _get_search_genome_version(families) - samples = _get_families_search_data(families, dataset_type=Sample.DATASET_TYPE_SV_CALLS) - - return _variant_lookup( - _sv_variant_lookup, user, variant_id, **kwargs, samples=samples, cache_key_suffix=user, - dataset_type=Sample.DATASET_TYPE_SV_CALLS, - ) - - -def _sv_variant_lookup(user, variant_id, dataset_type, samples, genome_version=None, sample_type=None, **kwargs): - if not sample_type: - raise InvalidSearchException('Sample type must be specified to look up a structural variant') - - data_type = f'{dataset_type}_{sample_type}' - - lookup_samples = samples.filter(sample_type=sample_type) - lookup_func = backend_specific_call(_raise_search_error('Lookup is disabled'), clickhouse_variant_lookup) - variant = lookup_func(user, variant_id, data_type, samples=lookup_samples, genome_version=genome_version, **kwargs) - variants = [variant] - - if variant['svType'] in {'DEL', 'DUP'}: - samples = samples.exclude(sample_type=sample_type) - search = { - 'parsed_locus': { - 'padded_interval': {'chrom': variant['chrom'], 'start': variant['pos'], 'end': variant['end'], 'padding': 0.2}, - }, - 'annotations': {'structural': [variant['svType'], f"gCNV_{variant['svType']}"]}, - **kwargs, - } - results = {} - _execute_search(samples, search, user, previous_search_results=results, genome_version=genome_version) - variants += results['all_results'] - - return variants - - def _get_search_cache_key(search_model, sort=None): return 'search_results__{}__{}'.format(search_model.guid, sort or XPOS_SORT_KEY) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index c8d6781bce..dca6bb1218 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -12,10 +12,11 @@ from math import ceil import re +from clickhouse_search.search import variant_lookup, sv_variant_lookup from reference_data.models import GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38 from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, get_search_samples, \ - variant_lookup, sv_variant_lookup, parse_variant_id + parse_variant_id, clickhouse_only from seqr.utils.search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY from seqr.utils.search.utils import InvalidSearchException from seqr.utils.xpos_utils import get_xpos @@ -552,19 +553,23 @@ def _parse_lookup_request(request): return parsed_variant_id, variant_id, kwargs @login_and_policies_required +@clickhouse_only def variant_lookup_handler(request): parsed_variant_id, variant_id, kwargs = _parse_lookup_request(request) is_sv = not parsed_variant_id - genome_version = kwargs.get('genome_version', GENOME_VERSION_GRCh38) + genome_version = kwargs.pop('genome_version', GENOME_VERSION_GRCh38) if is_sv: families = _all_genome_version_families( genome_version, request.user, ) if not families: raise PermissionDenied() - variants = sv_variant_lookup(request.user, variant_id, families, **kwargs) + sample_type = kwargs.get('sample_type') + if not sample_type: + raise InvalidSearchException('Sample type must be specified to look up a structural variant') + variants = sv_variant_lookup(request.user, variant_id, genome_version, families, sample_type, **kwargs) else: - variant = variant_lookup(request.user, parsed_variant_id, **kwargs) + variant = variant_lookup(request.user, parsed_variant_id, genome_version, **kwargs) variants = [variant] families = Family.objects.filter( guid__in=variant['familyGenotypes'], From 5a9aabb625864f093a0106ab5073ddb011b9fc3a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 15 Sep 2025 10:21:07 -0400 Subject: [PATCH 67/89] Revert "first pass move lookup to clickhosue" This reverts commit c147e10a4a353b999a8622c10416405aa0696192. --- clickhouse_search/search.py | 62 ++---------------------- seqr/utils/search/utils.py | 69 ++++++++++++++++++++++++++- seqr/views/apis/variant_search_api.py | 13 ++--- 3 files changed, 75 insertions(+), 69 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index f495ca6147..8916ed2054 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -1,6 +1,5 @@ from clickhouse_backend.models import ArrayField, StringField from collections import defaultdict -from datetime import timedelta from django.contrib.postgres.aggregates import ArrayAgg from django.core.exceptions import ObjectDoesNotExist from django.db import connections @@ -13,13 +12,13 @@ ArrayDistinct, ArrayMap from clickhouse_search.models import ENTRY_CLASS_MAP, ANNOTATIONS_CLASS_MAP, TRANSCRIPTS_CLASS_MAP, KEY_LOOKUP_CLASS_MAP, \ BaseClinvar, BaseAnnotationsMitoSnvIndel, BaseAnnotationsGRCh37SnvIndel, BaseAnnotationsSvGcnv -from reference_data.models import GeneConstraint, Omim, GENOME_VERSION_LOOKUP, GENOME_VERSION_GRCh37 +from reference_data.models import GeneConstraint, Omim, GENOME_VERSION_LOOKUP from seqr.models import Sample, PhenotypePrioritization, Individual from seqr.utils.logging_utils import SeqrLogger -from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_set_json from seqr.utils.search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \ PRIORITIZED_GENE_SORT, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS, RECESSIVE from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets +from settings import CLICKHOUSE_SERVICE_HOSTNAME logger = SeqrLogger(__name__) @@ -553,7 +552,7 @@ def _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples): variant = format_clickhouse_results([variant], genome_version)[0] return variant -def clickhouse_variant_lookup(user, variant_id, data_type, genome_version, samples=None, **kwargs): +def clickhouse_variant_lookup(user, variant_id, data_type, genome_version=None, samples=None, **kwargs): logger.info(f'Looking up variant {variant_id} with data type {data_type}', user) variant = _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples) @@ -608,61 +607,6 @@ def get_clickhouse_variant_by_id(variant_id, samples, genome_version, dataset_ty return None -def _variant_lookup(lookup_func, user, variant_id, genome_version, dataset_type, cache_key_suffix='', **kwargs): - _validate_dataset_type_genome_version(dataset_type, genome_version) - cache_key = f'variant_lookup_results__{variant_id}__{genome_version}__{cache_key_suffix}' - variant = safe_redis_get_json(cache_key) - if variant: - return variant - - variant = lookup_func(user, variant_id, dataset_type, genome_version, **kwargs) - safe_redis_set_json(cache_key, variant, expire=timedelta(weeks=2)) - return variant - - -def _validate_dataset_type_genome_version(dataset_type, genome_version): - if genome_version == GENOME_VERSION_GRCh37 and dataset_type != Sample.DATASET_TYPE_VARIANT_CALLS: - from seqr.utils.search.utils import InvalidSearchException - raise InvalidSearchException(f'{dataset_type} variants are not available for GRCh37') - - -def variant_lookup(user, parsed_variant_id, genome_version, **kwargs): - dataset_type = Sample.DATASET_TYPE_MITO_CALLS if parsed_variant_id[0].replace('chr', '').startswith('M') else Sample.DATASET_TYPE_VARIANT_CALLS - return _variant_lookup(clickhouse_variant_lookup, user, parsed_variant_id, genome_version, dataset_type, **kwargs) - - -def sv_variant_lookup(user, variant_id, genome_version, families, sample_type, **kwargs): - samples = _get_families_search_data(families, dataset_type=Sample.DATASET_TYPE_SV_CALLS) # TODO - - return _variant_lookup( - _sv_variant_lookup, user, variant_id, genome_version, dataset_type=Sample.DATASET_TYPE_SV_CALLS, - **kwargs, samples=samples, sample_type=sample_type, cache_key_suffix=user, - ) - - -def _sv_variant_lookup(user, variant_id, dataset_type, genome_version, samples, sample_type, **kwargs): - data_type = f'{dataset_type}_{sample_type}' - - lookup_samples = samples.filter(sample_type=sample_type) - variant = clickhouse_variant_lookup(user, variant_id, data_type, samples=lookup_samples, genome_version=genome_version, **kwargs) - variants = [variant] - - if variant['svType'] in {'DEL', 'DUP'}: - samples = samples.exclude(sample_type=sample_type) - search = { - 'parsed_locus': { - 'padded_interval': {'chrom': variant['chrom'], 'start': variant['pos'], 'end': variant['end'], 'padding': 0.2}, - }, - 'annotations': {'structural': [variant['svType'], f"gCNV_{variant['svType']}"]}, - **kwargs, - } - results = {} - _execute_search(samples, search, user, previous_search_results=results, genome_version=genome_version) # TODO - variants += results['all_results'] - - return variants - - def get_clickhouse_genotypes(project_guid, family_guids, genome_version, dataset_type, keys, samples): sample_data = _get_sample_data(samples.filter(individual__family__guid__in=family_guids))[dataset_type] entries = ENTRY_CLASS_MAP[genome_version][dataset_type].objects.filter( diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index f6411e05a7..e03416637d 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -5,7 +5,7 @@ from pyliftover.liftover import LiftOver from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, \ - get_clickhouse_cache_results, get_clickhouse_variant_by_id + get_clickhouse_cache_results, clickhouse_variant_lookup, get_clickhouse_variant_by_id from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 from seqr.models import Sample, Individual, Project from seqr.utils.logging_utils import SeqrLogger @@ -50,6 +50,12 @@ class InvalidSearchException(Exception): DATASET_TYPES_LOOKUP[DATASET_TYPE_NO_MITO] = [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_SV_CALLS] +def _raise_search_error(error): + def _wrapped(*args, **kwargs): + raise InvalidSearchException(error) + return _wrapped + + def es_only(func): def _wrapped(*args, **kwargs): if not es_backend_enabled(): @@ -146,6 +152,67 @@ def _get_clickhouse_variant_by_id(parsed_variant_id, variant_id, samples, genome ) +def _variant_lookup(lookup_func, user, variant_id, dataset_type, genome_version=None, cache_key_suffix='', **kwargs): + genome_version = genome_version or GENOME_VERSION_GRCh38 + _validate_dataset_type_genome_version(dataset_type, genome_version) + cache_key = f'variant_lookup_results__{variant_id}__{genome_version}__{cache_key_suffix}' + variant = safe_redis_get_json(cache_key) + if variant: + return variant + + variant = lookup_func(user, variant_id, dataset_type, genome_version=genome_version, **kwargs) + safe_redis_set_json(cache_key, variant, expire=timedelta(weeks=2)) + return variant + + +def _validate_dataset_type_genome_version(dataset_type, genome_version): + if genome_version == GENOME_VERSION_GRCh37 and dataset_type != Sample.DATASET_TYPE_VARIANT_CALLS: + raise InvalidSearchException(f'{dataset_type} variants are not available for GRCh37') + + +def variant_lookup(user, parsed_variant_id, **kwargs): + dataset_type = DATASET_TYPES_LOOKUP[_variant_ids_dataset_type([parsed_variant_id])][0] + lookup_func = backend_specific_call(_raise_search_error('Lookup is disabled'), clickhouse_variant_lookup) + return _variant_lookup(lookup_func, user, parsed_variant_id, **kwargs, dataset_type=dataset_type) + + +def sv_variant_lookup(user, variant_id, families, **kwargs): + _get_search_genome_version(families) + samples = _get_families_search_data(families, dataset_type=Sample.DATASET_TYPE_SV_CALLS) + + return _variant_lookup( + _sv_variant_lookup, user, variant_id, **kwargs, samples=samples, cache_key_suffix=user, + dataset_type=Sample.DATASET_TYPE_SV_CALLS, + ) + + +def _sv_variant_lookup(user, variant_id, dataset_type, samples, genome_version=None, sample_type=None, **kwargs): + if not sample_type: + raise InvalidSearchException('Sample type must be specified to look up a structural variant') + + data_type = f'{dataset_type}_{sample_type}' + + lookup_samples = samples.filter(sample_type=sample_type) + lookup_func = backend_specific_call(_raise_search_error('Lookup is disabled'), clickhouse_variant_lookup) + variant = lookup_func(user, variant_id, data_type, samples=lookup_samples, genome_version=genome_version, **kwargs) + variants = [variant] + + if variant['svType'] in {'DEL', 'DUP'}: + samples = samples.exclude(sample_type=sample_type) + search = { + 'parsed_locus': { + 'padded_interval': {'chrom': variant['chrom'], 'start': variant['pos'], 'end': variant['end'], 'padding': 0.2}, + }, + 'annotations': {'structural': [variant['svType'], f"gCNV_{variant['svType']}"]}, + **kwargs, + } + results = {} + _execute_search(samples, search, user, previous_search_results=results, genome_version=genome_version) + variants += results['all_results'] + + return variants + + def _get_search_cache_key(search_model, sort=None): return 'search_results__{}__{}'.format(search_model.guid, sort or XPOS_SORT_KEY) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index dca6bb1218..c8d6781bce 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -12,11 +12,10 @@ from math import ceil import re -from clickhouse_search.search import variant_lookup, sv_variant_lookup from reference_data.models import GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38 from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, get_search_samples, \ - parse_variant_id, clickhouse_only + variant_lookup, sv_variant_lookup, parse_variant_id from seqr.utils.search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY from seqr.utils.search.utils import InvalidSearchException from seqr.utils.xpos_utils import get_xpos @@ -553,23 +552,19 @@ def _parse_lookup_request(request): return parsed_variant_id, variant_id, kwargs @login_and_policies_required -@clickhouse_only def variant_lookup_handler(request): parsed_variant_id, variant_id, kwargs = _parse_lookup_request(request) is_sv = not parsed_variant_id - genome_version = kwargs.pop('genome_version', GENOME_VERSION_GRCh38) + genome_version = kwargs.get('genome_version', GENOME_VERSION_GRCh38) if is_sv: families = _all_genome_version_families( genome_version, request.user, ) if not families: raise PermissionDenied() - sample_type = kwargs.get('sample_type') - if not sample_type: - raise InvalidSearchException('Sample type must be specified to look up a structural variant') - variants = sv_variant_lookup(request.user, variant_id, genome_version, families, sample_type, **kwargs) + variants = sv_variant_lookup(request.user, variant_id, families, **kwargs) else: - variant = variant_lookup(request.user, parsed_variant_id, genome_version, **kwargs) + variant = variant_lookup(request.user, parsed_variant_id, **kwargs) variants = [variant] families = Family.objects.filter( guid__in=variant['familyGenotypes'], From 65377780659cc9bc0d9cc64be66ee80b46a1d7e5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 15 Sep 2025 11:17:24 -0400 Subject: [PATCH 68/89] remove family data from sv lookup and streamlibne calls --- clickhouse_search/search.py | 29 ++++++++++-- seqr/utils/search/utils.py | 66 ++++++--------------------- seqr/views/apis/variant_search_api.py | 42 +++++++---------- 3 files changed, 56 insertions(+), 81 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 8916ed2054..8c7ad5cdbd 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -526,7 +526,7 @@ def _get_sort_key(sort, gene_metadata): return lambda x: tuple(expr(x[0] if isinstance(x, list) else x) for expr in [*sort_expressions, lambda x: x[XPOS_SORT_KEY]]) -def _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples): +def _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples=None): entry_cls = ENTRY_CLASS_MAP[genome_version][data_type] annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][data_type] @@ -552,10 +552,12 @@ def _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples): variant = format_clickhouse_results([variant], genome_version)[0] return variant -def clickhouse_variant_lookup(user, variant_id, data_type, genome_version=None, samples=None, **kwargs): +def clickhouse_variant_lookup(user, variant_id, dataset_type, sample_type, genome_version): + is_sv = dataset_type == Sample.DATASET_TYPE_SV_CALLS + data_type = f'{dataset_type}_{sample_type}' if is_sv else dataset_type logger.info(f'Looking up variant {variant_id} with data type {data_type}', user) - variant = _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples) + variant = _clickhouse_variant_lookup(variant_id, genome_version, data_type) if variant: _add_liftover_genotypes(variant, data_type, variant_id) else: @@ -565,12 +567,29 @@ def clickhouse_variant_lookup(user, variant_id, data_type, genome_version=None, liftover_results = run_liftover(lifted_genome_version, variant_id[0], variant_id[1]) if liftover_results: lifted_id = (liftover_results[0], liftover_results[1], *variant_id[2:]) - variant = _clickhouse_variant_lookup(lifted_id, lifted_genome_version, data_type, samples) + variant = _clickhouse_variant_lookup(lifted_id, lifted_genome_version, data_type) if not variant: raise ObjectDoesNotExist('Variant not present in seqr') - return variant + variants = [variant] + + if is_sv and variant['svType'] in {'DEL', 'DUP'}: + other_sample_type, other_entry_class = next((dt, cls) for dt, cls in ENTRY_CLASS_MAP[genome_version].items() if dt != data_type and dt.startswith(Sample.DATASET_TYPE_SV_CALLS)) + other_annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][other_sample_type] + + padded_interval = {'chrom': variant['chrom'], 'start': variant['pos'], 'end': variant['end'], 'padding': 0.2} + entries = other_entry_class.objects.filter_locus(padded_interval=padded_interval) + entries = entries.result_values(sample_data=None) + results = other_annotations_cls.objects.subquery_join(entries) + results = results.filter_annotations( + results, + padded_interval=padded_interval, + annotations={'structural': [variant['svType'], f"gCNV_{variant['svType']}"]}, + ) + variants += list(results.result_values()) + + return variants def _add_liftover_genotypes(variant, data_type, variant_id): diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 25bb6bad9e..85b7c3c6e4 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -176,66 +176,30 @@ def get_variants_for_variant_ids(families, variant_ids, dataset_type=None, user= ) -def _variant_lookup(lookup_func, user, variant_id, dataset_type, genome_version=None, cache_key_suffix='', **kwargs): - genome_version = genome_version or GENOME_VERSION_GRCh38 - _validate_dataset_type_genome_version(dataset_type, genome_version) - cache_key = f'variant_lookup_results__{variant_id}__{genome_version}__{cache_key_suffix}' - variant = safe_redis_get_json(cache_key) - if variant: - return variant - - variant = lookup_func(user, variant_id, dataset_type, genome_version=genome_version, **kwargs) - safe_redis_set_json(cache_key, variant, expire=timedelta(weeks=2)) - return variant - - -def _validate_dataset_type_genome_version(dataset_type, genome_version): - if genome_version == GENOME_VERSION_GRCh37 and dataset_type != Sample.DATASET_TYPE_VARIANT_CALLS: - raise InvalidSearchException(f'{dataset_type} variants are not available for GRCh37') +def variant_lookup(user, variant_id, genome_version, sample_type=None): + backend_specific_call(_raise_search_error('Lookup is disabled'), lambda: None)() + cache_key = f'variant_lookup_results__{variant_id}__{genome_version}' + variants = safe_redis_get_json(cache_key) + if variants: + return variants -def variant_lookup(user, parsed_variant_id, **kwargs): + parsed_variant_id = parse_variant_id(variant_id) dataset_type = DATASET_TYPES_LOOKUP[_variant_ids_dataset_type([parsed_variant_id])][0] - lookup_func = backend_specific_call(_raise_search_error('Lookup is disabled'), clickhouse_variant_lookup) - return _variant_lookup(lookup_func, user, parsed_variant_id, **kwargs, dataset_type=dataset_type) + _validate_dataset_type_genome_version(dataset_type, sample_type, genome_version) + variants = clickhouse_variant_lookup(user, parsed_variant_id or variant_id, dataset_type, sample_type, genome_version) -def sv_variant_lookup(user, variant_id, families, **kwargs): - _get_search_genome_version(families) - samples = _get_families_search_data(families, dataset_type=Sample.DATASET_TYPE_SV_CALLS) - - return _variant_lookup( - _sv_variant_lookup, user, variant_id, **kwargs, samples=samples, cache_key_suffix=user, - dataset_type=Sample.DATASET_TYPE_SV_CALLS, - ) + safe_redis_set_json(cache_key, variants, expire=timedelta(weeks=2)) + return variants -def _sv_variant_lookup(user, variant_id, dataset_type, samples, genome_version=None, sample_type=None, **kwargs): - if not sample_type: +def _validate_dataset_type_genome_version(dataset_type, sample_type, genome_version): + if genome_version == GENOME_VERSION_GRCh37 and dataset_type != Sample.DATASET_TYPE_VARIANT_CALLS: + raise InvalidSearchException(f'{dataset_type} variants are not available for GRCh37') + if dataset_type == Sample.DATASET_TYPE_SV_CALLS and not sample_type: raise InvalidSearchException('Sample type must be specified to look up a structural variant') - data_type = f'{dataset_type}_{sample_type}' - - lookup_samples = samples.filter(sample_type=sample_type) - lookup_func = backend_specific_call(_raise_search_error('Lookup is disabled'), clickhouse_variant_lookup) - variant = lookup_func(user, variant_id, data_type, samples=lookup_samples, genome_version=genome_version, **kwargs) - variants = [variant] - - if variant['svType'] in {'DEL', 'DUP'}: - samples = samples.exclude(sample_type=sample_type) - search = { - 'parsed_locus': { - 'padded_interval': {'chrom': variant['chrom'], 'start': variant['pos'], 'end': variant['end'], 'padding': 0.2}, - }, - 'annotations': {'structural': [variant['svType'], f"gCNV_{variant['svType']}"]}, - **kwargs, - } - results = {} - _execute_search(samples, search, user, previous_search_results=results, genome_version=genome_version) - variants += results['all_results'] - - return variants - def _get_search_cache_key(search_model, sort=None): return 'search_results__{}__{}'.format(search_model.guid, sort or XPOS_SORT_KEY) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index c8d6781bce..f80c7dcd5e 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -544,32 +544,22 @@ def _flatten_variants(variants): flattened_variants.append(variant) return flattened_variants -def _parse_lookup_request(request): - kwargs = {_to_snake_case(k): v for k, v in request.GET.items()} - - variant_id = kwargs.pop('variant_id') - parsed_variant_id = parse_variant_id(variant_id) - return parsed_variant_id, variant_id, kwargs @login_and_policies_required def variant_lookup_handler(request): - parsed_variant_id, variant_id, kwargs = _parse_lookup_request(request) - is_sv = not parsed_variant_id - genome_version = kwargs.get('genome_version', GENOME_VERSION_GRCh38) - if is_sv: - families = _all_genome_version_families( - genome_version, request.user, - ) - if not families: - raise PermissionDenied() - variants = sv_variant_lookup(request.user, variant_id, families, **kwargs) - else: - variant = variant_lookup(request.user, parsed_variant_id, **kwargs) - variants = [variant] - families = Family.objects.filter( - guid__in=variant['familyGenotypes'], - project__guid__in=get_project_guids_user_can_view(request.user, limit_data_manager=True), - ) + variant_id = request.GET.get('variantId') + genome_version = request.GET.get('genomeVersion') or GENOME_VERSION_GRCh38 + variants = variant_lookup(request.user, variant_id, genome_version, sample_type=request.GET.get('sampleType')) + + family_guids = set() + for variant in variants: + family_guids.update(variant['familyGenotypes'].keys()) + + families = Family.objects.filter( + guid__in=family_guids, + project__guid__in=get_project_guids_user_can_view(request.user, limit_data_manager=True), + ) + for variant in variants: variant['familyGuids'] = list(families.values_list('guid', flat=True)) saved_variants, _ = _get_saved_variant_models(variants, None) if families else (None, None) @@ -579,7 +569,7 @@ def variant_lookup_handler(request): ) response['variants'] = variants - if not is_sv: + for variant in variants: _update_lookup_variant(variant, response) return create_json_response(response) @@ -641,7 +631,9 @@ def _update_lookup_variant(variant, response): @login_and_policies_required def vlm_lookup_handler(request): - parsed_variant_id, _, kwargs = _parse_lookup_request(request) + kwargs = {_to_snake_case(k): v for k, v in request.GET.items()} + variant_id = kwargs.pop('variant_id') + parsed_variant_id = parse_variant_id(variant_id) if parsed_variant_id: invalid_alleles = [f'"{allele}"' for allele in parsed_variant_id[2:] if not re.fullmatch(r'[ATCG]+', allele)] if invalid_alleles: From 951b6e287ce217abd02e30a85eb97b228d11dd72 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 15 Sep 2025 11:21:38 -0400 Subject: [PATCH 69/89] clean up --- clickhouse_search/managers.py | 2 +- clickhouse_search/search.py | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 1f8528ed7c..d98c2a32b4 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -724,7 +724,7 @@ def _join_annotations(self, entries): return entries - def result_values(self, sample_data): + def result_values(self, sample_data=None): entries = self._join_annotations(self) if sample_data: return self._search_call_data(entries, sample_data) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 8c7ad5cdbd..be9aa2e258 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -579,11 +579,8 @@ def clickhouse_variant_lookup(user, variant_id, dataset_type, sample_type, genom other_annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][other_sample_type] padded_interval = {'chrom': variant['chrom'], 'start': variant['pos'], 'end': variant['end'], 'padding': 0.2} - entries = other_entry_class.objects.filter_locus(padded_interval=padded_interval) - entries = entries.result_values(sample_data=None) - results = other_annotations_cls.objects.subquery_join(entries) - results = results.filter_annotations( - results, + entries = other_entry_class.objects.filter_locus(padded_interval=padded_interval).result_values() + results = other_annotations_cls.objects.subquery_join(entries).search( padded_interval=padded_interval, annotations={'structural': [variant['svType'], f"gCNV_{variant['svType']}"]}, ) From 4378922dc5a38640d6687efb354e065a170fbf19 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 15 Sep 2025 11:33:51 -0400 Subject: [PATCH 70/89] remove generic padded interval handling --- clickhouse_search/managers.py | 22 +++++++++++----------- clickhouse_search/search.py | 11 +++++++---- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index d98c2a32b4..7cf4477d1b 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -413,7 +413,7 @@ def _get_in_silico_score_q(self, score, value): except ValueError: return Q(**{score_column: value}) - def filter_annotations(self, results, annotations=None, pathogenicity=None, exclude=None, gene_ids=None, intervals=None, exclude_intervals=False, padded_interval=None, **kwargs): + def filter_annotations(self, results, annotations=None, pathogenicity=None, exclude=None, gene_ids=None, intervals=None, exclude_intervals=False, padded_interval_end=None, **kwargs): transcript_field = self.transcript_field if self.model.GENOTYPE_OVERRIDE_FIELDS: results = results.annotate(**{ @@ -433,10 +433,10 @@ def filter_annotations(self, results, annotations=None, pathogenicity=None, excl for interval_q in interval_qs: gene_q |= interval_q results = results.filter(gene_q) - elif (interval_qs or padded_interval) and hasattr(self.model, 'end'): - if padded_interval: - padding = int((padded_interval['end'] - padded_interval['start']) * padded_interval['padding']) - interval_qs = [Q(end__range=(padded_interval['end'] - padding, padded_interval['end'] + padding))] + elif (interval_qs or padded_interval_end) and hasattr(self.model, 'end'): + if padded_interval_end: + end, padding = padded_interval_end + interval_qs = [Q(end__range=(end - padding, end + padding))] interval_q = interval_qs[0] for q in interval_qs[1:]: interval_q |= q @@ -1043,18 +1043,14 @@ def _family_passes_expression(pass_field): mapped_expression='x.1', output_field=models.ArrayField(models.StringField()), ) - def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=None, gene_ids=None, variant_ids=None, padded_interval=None, require_gene_filter=False, **kwargs): + def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=None, gene_ids=None, variant_ids=None, require_gene_filter=False, **kwargs): entries = self if variant_ids: # although technically redundant, the interval query is applied to the entries table before join and reduces the join size, # while the full variant_id filter is applied to the annotation table after the join intervals = [(chrom, pos, pos) for chrom, pos, _, _ in variant_ids] - if padded_interval: - pos = padded_interval['start'] - padding = int((padded_interval['end'] - pos) * padded_interval['padding']) - intervals = [(padded_interval['chrom'], max(pos - padding, MIN_POS), min(pos + padding, MAX_POS))] - elif 'cn' in self.call_fields: + if 'cn' in self.call_fields: # SV interval filtering occurs after joining on annotations to correctly incorporate end position if exclude_intervals: intervals = None @@ -1096,6 +1092,10 @@ def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=N filter_func = entries.exclude if exclude_intervals else entries.filter return filter_func(locus_q) + def search_padded_interval(self, chrom, pos, padding): + interval_q = self._interval_query(chrom, max(pos - padding, MIN_POS), min(pos + padding, MAX_POS)) + return self.filter(interval_q).result_values() + @staticmethod def _interval_query(chrom, start, end): return Q(xpos__range=(get_xpos(chrom, start), get_xpos(chrom, end))) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index be9aa2e258..315a45fded 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -575,13 +575,16 @@ def clickhouse_variant_lookup(user, variant_id, dataset_type, sample_type, genom variants = [variant] if is_sv and variant['svType'] in {'DEL', 'DUP'}: - other_sample_type, other_entry_class = next((dt, cls) for dt, cls in ENTRY_CLASS_MAP[genome_version].items() if dt != data_type and dt.startswith(Sample.DATASET_TYPE_SV_CALLS)) + other_sample_type, other_entry_class = next( + (dt, cls) for dt, cls in ENTRY_CLASS_MAP[genome_version].items() + if dt != data_type and dt.startswith(Sample.DATASET_TYPE_SV_CALLS) + ) other_annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][other_sample_type] - padded_interval = {'chrom': variant['chrom'], 'start': variant['pos'], 'end': variant['end'], 'padding': 0.2} - entries = other_entry_class.objects.filter_locus(padded_interval=padded_interval).result_values() + padding = int((variant['end'] - variant['pos']) * 0.2) + entries = other_entry_class.objects.search_padded_interval(variant['chrom'], variant['pos'], padding) results = other_annotations_cls.objects.subquery_join(entries).search( - padded_interval=padded_interval, + padded_interval_end=(variant['end'], padding), annotations={'structural': [variant['svType'], f"gCNV_{variant['svType']}"]}, ) variants += list(results.result_values()) From 2507956dd22b5b7048a37d02a503b0c45f28e751 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 15 Sep 2025 13:38:04 -0400 Subject: [PATCH 71/89] fix clickhouse tests --- clickhouse_search/search_tests.py | 41 ++++++++++++------------- clickhouse_search/test_utils.py | 32 +++++++++++++++++++ seqr/utils/search/search_utils_tests.py | 2 +- seqr/views/apis/variant_search_api.py | 2 +- 4 files changed, 53 insertions(+), 24 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 6ffc0aaaaa..d780415bd7 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -14,11 +14,12 @@ SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4, SV_GENE_COUNTS, NEW_SV_FILTER, GCNV_VARIANT1, GCNV_VARIANT2, \ GCNV_VARIANT3, GCNV_VARIANT4, GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_GENE_COUNTS, \ MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, VARIANT_LOOKUP_VARIANT, \ - MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, format_cached_variant + MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT_3, \ + format_cached_variant from reference_data.models import Omim from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.utils.search.search_utils_tests import SearchTestHelper -from seqr.utils.search.utils import query_variants, variant_lookup, sv_variant_lookup, get_variant_query_gene_counts, get_single_variant, InvalidSearchException +from seqr.utils.search.utils import query_variants, variant_lookup, get_variant_query_gene_counts, get_single_variant, InvalidSearchException from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin @@ -664,50 +665,46 @@ def test_variant_lookup(self, mock_liftover): mock_convert_coordinate = mock_liftover.return_value.convert_coordinate mock_convert_coordinate.side_effect = lambda chrom, pos: [(chrom, pos + 10000)] - variant = variant_lookup(self.user, ('1', 10439, 'AC', 'A')) - self._assert_expected_variants([variant], [VARIANT_LOOKUP_VARIANT]) + variants = variant_lookup(self.user, '1-10439-AC-A', '38') + self._assert_expected_variants(variants, [VARIANT_LOOKUP_VARIANT]) with self.assertRaises(ObjectDoesNotExist) as cm: - variant_lookup(self.user, ('1', 91511686, 'TCA', 'G')) + variant_lookup(self.user, '1-91511686-TCA-G', '38') self.assertEqual(str(cm.exception), 'Variant not present in seqr') - variant = variant_lookup(self.user, ('7', 143270172, 'A', 'G'), genome_version='37') + variants = variant_lookup(self.user, '7-143270172-A-G', '37') grch37_lookup_variant = { **{k: v for k, v in GRCH37_VARIANT.items() if k not in {'familyGuids', 'genotypes'}}, 'familyGenotypes': {GRCH37_VARIANT['familyGuids'][0]: sorted([ {k: v for k, v in g.items() if k != 'individualGuid'} for g in GRCH37_VARIANT['genotypes'].values() ], key=lambda x: x['sampleId'], reverse=True)}, } - self._assert_expected_variants([variant], [grch37_lookup_variant]) + self._assert_expected_variants(variants, [grch37_lookup_variant]) # Lookup works if variant is only present on a different build - variant = variant_lookup(self.user, ('7', 143260172, 'A', 'G')) - self._assert_expected_variants([variant], [grch37_lookup_variant]) + variants = variant_lookup(self.user, '7-143260172-A-G', '38') + self._assert_expected_variants(variants, [grch37_lookup_variant]) mock_liftover.assert_called_with('hg38', 'hg19') mock_convert_coordinate.assert_called_with('chr7', 143260172) - variant = variant_lookup(self.user, ('M', 4429, 'G', 'A'), genome_version='38') - self._assert_expected_variants([variant], [{ + variants = variant_lookup(self.user, 'M-4429-G-A', '38') + self._assert_expected_variants(variants, [{ **{k: v for k, v in MITO_VARIANT1.items() if k not in {'familyGuids', 'genotypes'}}, 'familyGenotypes': {MITO_VARIANT1['familyGuids'][0]: [ {k: v for k, v in g.items() if k != 'individualGuid'} for g in MITO_VARIANT1['genotypes'].values() ]}, }]) - families = Family.objects.all() - variants = sv_variant_lookup(self.user, 'phase2_DEL_chr14_4640', families, sample_type='WGS') - self._assert_expected_variants(variants, [SV_VARIANT4, GCNV_VARIANT4]) + variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS') + self.maxDiff = None + self._assert_expected_variants(variants, [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT]) # reciprocal overlap does not meet the threshold for smaller events - variants = sv_variant_lookup(self.user, 'suffix_140608_DUP', families, sample_type='WES') - self._assert_expected_variants(variants, [GCNV_VARIANT4]) - - variants = sv_variant_lookup(self.user, 'suffix_140593_DUP', families, sample_type='WES') - self._assert_expected_variants(variants, [GCNV_VARIANT3]) + variants = variant_lookup(self.user, 'suffix_140608_DUP', '38', sample_type='WES') + self._assert_expected_variants(variants, [GCNV_LOOKUP_VARIANT]) - with self.assertRaises(ObjectDoesNotExist) as cm: - variant_lookup(self.user, ('5', 143270172, 'A', 'G')) - self.assertEqual(str(cm.exception), 'Variant not present in seqr') + variants = variant_lookup(self.user, 'suffix_140593_DUP', '38', sample_type='WES') + self._assert_expected_variants(variants, [GCNV_LOOKUP_VARIANT_3]) def test_get_single_variant(self): self._set_single_family_search() diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 3bfe4c220a..91c001d888 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1058,8 +1058,40 @@ }], } } +SV_LOOKUP_VARIANT = { + **SV_VARIANT4, + 'familyGenotypes': { + SV_VARIANT4['familyGuids'][0]: [{k: v for k, v in g.items() if k != 'individualGuid'} for g in SV_VARIANT4['genotypes'].values()], + }, +} +GCNV_LOOKUP_VARIANT = { + **GCNV_VARIANT4, + 'familyGenotypes': { + GCNV_VARIANT4['familyGuids'][0]: [{k: v for k, v in g.items() if k != 'individualGuid'} for g in GCNV_VARIANT4['genotypes'].values()], + 'F000002_2_x': [ + {**{k: v for k, v in g.items() if k != 'individualGuid'}, 'familyGuid': 'F000002_2_x'} + for individual_guid, g in GCNV_VARIANT4['genotypes'].items() if individual_guid != 'I000005_hg00732' + ], + }, +} +GCNV_LOOKUP_VARIANT_3 = { + **MULTI_PROJECT_GCNV_VARIANT3, + 'familyGenotypes': { + **{family_guid: [ + {k: v for k, v in g.items() if k != 'individualGuid'} for g in MULTI_PROJECT_GCNV_VARIANT3['genotypes'].values() + if g['familyGuid'] == family_guid] for family_guid in MULTI_PROJECT_GCNV_VARIANT3['familyGuids'] + }, + 'F000002_2_x': [ + {**{k: v for k, v in g.items() if k != 'individualGuid'}, 'familyGuid': 'F000002_2_x'} + for individual_guid, g in GCNV_VARIANT3['genotypes'].items() if individual_guid != 'I000005_hg00732' + ], + }, +} for k in {'familyGuids', 'genotypes'}: VARIANT_LOOKUP_VARIANT.pop(k) + SV_LOOKUP_VARIANT.pop(k) + GCNV_LOOKUP_VARIANT.pop(k) + GCNV_LOOKUP_VARIANT_3.pop(k) PROJECT_4_COMP_HET_VARIANT = { 'key': 22, diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 80bf5b7b08..ce1d5b703e 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -9,7 +9,7 @@ from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets from seqr.utils.search.utils import get_single_variant, get_variants_for_variant_ids, get_variant_query_gene_counts, \ - query_variants, variant_lookup, sv_variant_lookup, InvalidSearchException + query_variants, variant_lookup, InvalidSearchException from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin, PARSED_VARIANTS, PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT, GENE_FIELDS diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index f80c7dcd5e..ba27e43753 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -15,7 +15,7 @@ from reference_data.models import GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38 from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, get_search_samples, \ - variant_lookup, sv_variant_lookup, parse_variant_id + variant_lookup, parse_variant_id from seqr.utils.search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY from seqr.utils.search.utils import InvalidSearchException from seqr.utils.xpos_utils import get_xpos From 8e86598f092b8b989de8b0a0a4939aa02fb6b405 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 15 Sep 2025 13:54:42 -0400 Subject: [PATCH 72/89] fix lookup test --- clickhouse_search/search_tests.py | 1 - seqr/utils/search/search_utils_tests.py | 20 ++++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index d780415bd7..8aa7186072 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -696,7 +696,6 @@ def test_variant_lookup(self, mock_liftover): }]) variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS') - self.maxDiff = None self._assert_expected_variants(variants, [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT]) # reciprocal overlap does not meet the threshold for smaller events diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index ce1d5b703e..41a7c9492e 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -65,17 +65,17 @@ def set_up(self): self.search_samples = list(self.affected_search_samples) + list(self.non_affected_search_samples) def test_variant_lookup(self, mock_variant_lookup): - mock_variant_lookup.return_value = VARIANT_LOOKUP_VARIANT - variant = variant_lookup(self.user, ('1', 10439, 'AC', 'A'), genome_version='38') - self.assertDictEqual(variant, VARIANT_LOOKUP_VARIANT) - mock_variant_lookup.assert_called_with(self.user, ('1', 10439, 'AC', 'A'), 'SNV_INDEL', genome_version='38') - cache_key = "variant_lookup_results__('1', 10439, 'AC', 'A')__38__" - self.assert_cached_results(variant, cache_key=cache_key) + mock_variant_lookup.return_value = [VARIANT_LOOKUP_VARIANT] + variants = variant_lookup(self.user, '1-10439-AC-A', '38') + self.assertListEqual(variants, [VARIANT_LOOKUP_VARIANT]) + mock_variant_lookup.assert_called_with(self.user, ('1', 10439, 'AC', 'A'), 'SNV_INDEL', None, '38') + cache_key = "variant_lookup_results__1-10439-AC-A__38" + self.assert_cached_results(variants, cache_key=cache_key) mock_variant_lookup.reset_mock() - self.set_cache(variant) - cached_variant = variant_lookup(self.user, ('1', 10439, 'AC', 'A'), genome_version='38') - self.assertDictEqual(variant, cached_variant) + self.set_cache(variants) + cached_variant = variant_lookup(self.user, '1-10439-AC-A', '38') + self.assertListEqual(variants, cached_variant) mock_variant_lookup.assert_not_called() self.mock_redis.get.assert_called_with(cache_key) @@ -84,7 +84,7 @@ def test_sv_variant_lookup(self, mock_sv_variant_lookup, mock_get_variants): def _mock_get_variants(samples, search, user, previous_search_results, genome_version, **kwargs): previous_search_results['all_results'] = [SV_VARIANT1] mock_get_variants.side_effect = _mock_get_variants - variants = sv_variant_lookup(self.user, 'phase2_DEL_chr14_4640', self.families, genome_version='38', sample_type='WGS') + variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS') self.assertListEqual(variants, [SV_VARIANT4, SV_VARIANT1]) mock_sv_variant_lookup.assert_called_with( self.user, 'phase2_DEL_chr14_4640', 'SV_WGS', genome_version='38', samples=mock.ANY) From 5ce7c0b1c79295d0dc2d04fd362e2a6b796cb151 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 15 Sep 2025 13:59:51 -0400 Subject: [PATCH 73/89] fix sv lookup test --- seqr/utils/search/search_utils_tests.py | 41 +++++++------------------ 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 41a7c9492e..7a88bbe1a6 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -5,7 +5,7 @@ import mock from clickhouse_search.test_utils import VARIANT1, VARIANT2, VARIANT3, VARIANT4, PROJECT_2_VARIANT2, format_cached_variant, \ - GENE_COUNTS, VARIANT_LOOKUP_VARIANT, SV_VARIANT4, SV_VARIANT1 + GENE_COUNTS, VARIANT_LOOKUP_VARIANT, SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, SV_VARIANT1 from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets from seqr.utils.search.utils import get_single_variant, get_variants_for_variant_ids, get_variant_query_gene_counts, \ @@ -79,30 +79,21 @@ def test_variant_lookup(self, mock_variant_lookup): mock_variant_lookup.assert_not_called() self.mock_redis.get.assert_called_with(cache_key) - def test_sv_variant_lookup(self, mock_sv_variant_lookup, mock_get_variants): - mock_sv_variant_lookup.return_value = SV_VARIANT4 - def _mock_get_variants(samples, search, user, previous_search_results, genome_version, **kwargs): - previous_search_results['all_results'] = [SV_VARIANT1] - mock_get_variants.side_effect = _mock_get_variants + self.set_cache(None) + mock_variant_lookup.reset_mock() + mock_variant_lookup.return_value = [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT] variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS') - self.assertListEqual(variants, [SV_VARIANT4, SV_VARIANT1]) - mock_sv_variant_lookup.assert_called_with( - self.user, 'phase2_DEL_chr14_4640', 'SV_WGS', genome_version='38', samples=mock.ANY) - mock_get_variants.assert_called_with(mock.ANY, { - 'parsed_locus': {'padded_interval': {'chrom': '17', 'start': 38719997, 'end': 38737237, 'padding': 0.2}}, - 'annotations': {'structural': ['DEL', 'gCNV_DEL']}, - }, self.user, mock.ANY, '38') - cache_key = 'variant_lookup_results__phase2_DEL_chr14_4640__38__test_user' + self.assertListEqual(variants, [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT]) + mock_variant_lookup.assert_called_with( + self.user, 'phase2_DEL_chr14_4640', 'SV', 'WGS', '38') + cache_key = 'variant_lookup_results__phase2_DEL_chr14_4640__38' self.assert_cached_results(variants, cache_key=cache_key) - expected_samples = {s for s in self.search_samples if s.guid in SV_SAMPLES} - self.assertSetEqual(set(mock_get_variants.call_args.args[0]), expected_samples) - self.assertListEqual(list(mock_sv_variant_lookup.call_args.kwargs['samples']), []) - mock_sv_variant_lookup.reset_mock() + mock_variant_lookup.reset_mock() self.set_cache(variants) - cached_variant = sv_variant_lookup(self.user, 'phase2_DEL_chr14_4640', self.families, genome_version='38') + cached_variant = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS') self.assertListEqual(variants, cached_variant) - mock_sv_variant_lookup.assert_not_called() + mock_variant_lookup.assert_not_called() self.mock_redis.get.assert_called_with(cache_key) def test_get_single_variant(self, mock_get_variants_for_ids): @@ -579,11 +570,6 @@ def test_variant_lookup(self, *args, **kwargs): super().test_variant_lookup(mock.MagicMock()) self.assertEqual(str(cm.exception), 'Lookup is disabled') - def test_sv_variant_lookup(self, *args, **kwargs): - with self.assertRaises(InvalidSearchException) as cm: - super().test_sv_variant_lookup(mock.MagicMock(), mock.MagicMock()) - self.assertEqual(str(cm.exception), 'Lookup is disabled') - @mock.patch('seqr.utils.search.utils.get_es_variants_for_variant_ids') def test_get_single_variant(self, mock_get_variants_for_ids): mock_get_variants_for_ids.return_value = [PARSED_VARIANTS[0]] @@ -747,8 +733,3 @@ def test_cached_get_variant_query_gene_counts(self): @mock.patch('seqr.utils.search.utils.clickhouse_variant_lookup') def test_variant_lookup(self, mock_call): super().test_variant_lookup(mock_call) - - @mock.patch('seqr.utils.search.utils.get_clickhouse_variants') - @mock.patch('seqr.utils.search.utils.clickhouse_variant_lookup') - def test_sv_variant_lookup(self, mock_sv_variant_lookup, mock_get_variants): - super().test_sv_variant_lookup(mock_sv_variant_lookup, mock_get_variants) From fedd6a8bb74e43d7b18c4124940ec17a194fb248 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 15 Sep 2025 14:03:03 -0400 Subject: [PATCH 74/89] test dataset type validation --- seqr/utils/search/search_utils_tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 7a88bbe1a6..7b3fc58e89 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -81,6 +81,14 @@ def test_variant_lookup(self, mock_variant_lookup): self.set_cache(None) mock_variant_lookup.reset_mock() + with self.assertRaises(InvalidSearchException) as cm: + variant_lookup(self.user, 'phase2_DEL_chr14_4640', '37') + self.assertEqual(str(cm.exception), 'SV variants are not available for GRCh37') + + with self.assertRaises(InvalidSearchException) as cm: + variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38') + self.assertEqual(str(cm.exception), 'Sample type must be specified to look up a structural variant') + mock_variant_lookup.return_value = [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT] variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS') self.assertListEqual(variants, [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT]) From 2574392693ad3e5d049e522d9e98b5b63bfcf821 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 15 Sep 2025 16:27:28 -0400 Subject: [PATCH 75/89] test sv lookup with no access --- seqr/views/apis/variant_search_api.py | 11 +- seqr/views/apis/variant_search_api_tests.py | 131 ++++++++++++++++---- 2 files changed, 110 insertions(+), 32 deletions(-) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index ba27e43753..4e0dff6ea1 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -569,17 +569,16 @@ def variant_lookup_handler(request): ) response['variants'] = variants + individual_guid_map = { + (i['familyGuid'], i['individualId']): i['individualGuid'] for i in response['individualsByGuid'].values() + } for variant in variants: - _update_lookup_variant(variant, response) + _update_lookup_variant(variant, response, individual_guid_map) return create_json_response(response) -def _update_lookup_variant(variant, response): - individual_guid_map = { - (i['familyGuid'], i['individualId']): i['individualGuid'] for i in response['individualsByGuid'].values() - } - +def _update_lookup_variant(variant, response, individual_guid_map): no_access_families = set(variant['familyGenotypes']) - set(variant['familyGuids']) individual_summary_map = { (i.pop('family__guid'), i.pop('individual_id')): (i.pop('guid'), i) diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 2d3688a009..5bbf16854b 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -7,7 +7,8 @@ from django.urls.base import reverse from elasticsearch.exceptions import ConnectionTimeout, TransportError -from clickhouse_search.test_utils import VARIANT2, VARIANT3, VARIANT_LOOKUP_VARIANT, GCNV_VARIANT1, SV_VARIANT1 +from clickhouse_search.test_utils import VARIANT2, VARIANT3, VARIANT_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, \ + SV_LOOKUP_VARIANT, SV_VARIANT4, GCNV_VARIANT4 from seqr.models import VariantSearchResults, LocusList, Project, VariantSearch from seqr.utils.search.utils import InvalidSearchException from seqr.utils.search.elasticsearch.es_utils import InvalidIndexException @@ -903,7 +904,7 @@ def _assert_expected_single_variant_results_context(self, response_json, omit_fi @mock.patch('seqr.views.apis.variant_search_api.variant_lookup') def test_variant_lookup(self, mock_variant_lookup): response_variant = deepcopy(VARIANT_LOOKUP_VARIANT) - mock_variant_lookup.side_effect = lambda *args, **kwargs: deepcopy(response_variant) + mock_variant_lookup.side_effect = lambda *args, **kwargs: [deepcopy(response_variant)] url = f'{reverse(variant_lookup_handler)}?variantId=1-10439-AC-A&genomeVersion=38' self.check_require_login(url) @@ -936,13 +937,7 @@ def test_variant_lookup(self, mock_variant_lookup): }, } del expected_variant['familyGenotypes'] - expected_body = { - **{k: {} for k in self.EXPECTED_SEARCH_RESPONSE if k not in { - 'searchedVariants', 'search', 'variantNotesByGuid', 'variantTagsByGuid', 'variantFunctionalDataByGuid', 'transcriptsById', - }}, - **{k: {} for k in EXPECTED_SEARCH_FAMILY_CONTEXT}, - 'projectsByGuid': {}, - 'individualsByGuid': { + expected_body = self._expected_lookup_body({ 'I0_F0_1-10439-AC-A': { 'affected': 'N', 'familyGuid': 'F0_1-10439-AC-A', 'features': [], 'individualGuid': 'I0_F0_1-10439-AC-A', 'sex': 'F', @@ -969,12 +964,10 @@ def test_variant_lookup(self, mock_variant_lookup): 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', }, }, - 'variants': [expected_variant], - } - if 'totalSampleCounts' in self.EXPECTED_SEARCH_RESPONSE: - expected_body['totalSampleCounts'] = self.EXPECTED_SEARCH_RESPONSE['totalSampleCounts'] + [expected_variant], + ) self.assertDictEqual(response.json(), expected_body) - mock_variant_lookup.assert_called_with(self.no_access_user, ('1', 10439, 'AC', 'A'), genome_version='38') + mock_variant_lookup.assert_called_with(self.no_access_user, '1-10439-AC-A', '38', sample_type=None) response_variant['transcripts'] = VARIANTS[0]['transcripts'] expected_variant['transcripts'] = VARIANTS[0]['transcripts'] @@ -1046,29 +1039,115 @@ def test_variant_lookup(self, mock_variant_lookup): self.assertDictEqual(response.json(), expected_body) mock_variant_lookup.assert_called_with( - self.manager_user, ('1', 10439, 'AC', 'A'), genome_version='37', + self.manager_user, '1-10439-AC-A', '37', sample_type=None, ) - @mock.patch('seqr.views.apis.variant_search_api.sv_variant_lookup') + def _expected_lookup_body(self, individuals_by_guid, variants): + expected_body = { + **{k: {} for k in self.EXPECTED_SEARCH_RESPONSE if k not in { + 'searchedVariants', 'search', 'variantNotesByGuid', 'variantTagsByGuid', 'variantFunctionalDataByGuid', + 'transcriptsById', + }}, + **{k: {} for k in EXPECTED_SEARCH_FAMILY_CONTEXT}, + 'projectsByGuid': {}, + 'individualsByGuid': individuals_by_guid, + 'variants': variants, + } + if 'totalSampleCounts' in self.EXPECTED_SEARCH_RESPONSE: + expected_body['totalSampleCounts'] = self.EXPECTED_SEARCH_RESPONSE['totalSampleCounts'] + return expected_body + + @mock.patch('seqr.views.apis.variant_search_api.variant_lookup') def test_sv_variant_lookup(self, mock_variant_lookup): - variants = [{**variant, 'familyGuids': ['F000001_1']} for variant in [SV_VARIANT1, GCNV_VARIANT1]] - mock_variant_lookup.return_value = variants + mock_variant_lookup.return_value = deepcopy([ + {**GCNV_LOOKUP_VARIANT, 'familyGenotypes': {k: v for k, v in GCNV_LOOKUP_VARIANT['familyGenotypes'].items() if k != 'F000002_2_x'}} , + SV_LOOKUP_VARIANT, + ]) url = f'{reverse(variant_lookup_handler)}?variantId=phase2_DEL_chr14_4640&genomeVersion=37&sampleType=WGS' - self.check_collaborator_login(url) + self.check_require_login(url) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self._assert_expected_single_variant_results_context( - response.json(), variants=variants, omit_fields={'searchedVariants'}, no_metadata=True, + expected_gcnv_variant = { + **GCNV_VARIANT4, + 'familyGuids': [], + 'lookupFamilyGuids': ['F0_suffix_140608_DUP'], + 'genotypes': { + 'I0_F0_suffix_140608_DUP': { + k: v for k, v in GCNV_VARIANT4['genotypes']['I000004_hg00731'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I1_F0_suffix_140608_DUP': { + k: v for k, v in GCNV_VARIANT4['genotypes']['I000005_hg00732'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I2_F0_suffix_140608_DUP': { + k: v for k, v in GCNV_VARIANT4['genotypes']['I000006_hg00733'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + }, + } + expected_sv_variant = { + **SV_VARIANT4, + 'familyGuids': [], + 'lookupFamilyGuids': ['F0_phase2_DEL_chr14_4640'], + 'genotypes': { + 'I0_F0_phase2_DEL_chr14_4640': { + k: v for k, v in SV_VARIANT4['genotypes']['I000018_na21234'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I1_F0_phase2_DEL_chr14_4640': { + k: v for k, v in SV_VARIANT4['genotypes']['I000019_na21987'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I2_F0_phase2_DEL_chr14_4640': { + k: v for k, v in SV_VARIANT4['genotypes']['I000021_na21654'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + }, + } + expected_body = self._expected_lookup_body({ + 'I0_F0_phase2_DEL_chr14_4640': { + 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], + 'individualGuid': 'I0_F0_phase2_DEL_chr14_4640', 'sex': 'F', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I0_F0_suffix_140608_DUP': { + 'affected': 'A', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I0_F0_suffix_140608_DUP', + 'features': [{'category': 'HP:0000707', 'label': '1 terms'}, {'category': 'HP:0001626', 'label': '1 terms'}], + 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I1_F0_phase2_DEL_chr14_4640': { + 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], + 'individualGuid': 'I1_F0_phase2_DEL_chr14_4640', 'sex': 'M', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I1_F0_suffix_140608_DUP': { + 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I1_F0_suffix_140608_DUP', + 'sex': 'M','features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I2_F0_phase2_DEL_chr14_4640': { + 'affected': 'N', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], + 'individualGuid': 'I2_F0_phase2_DEL_chr14_4640', 'sex': 'F', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I2_F0_suffix_140608_DUP': { + 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I2_F0_suffix_140608_DUP', + 'sex': 'F', 'features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + },[expected_gcnv_variant, expected_sv_variant]) + self.assertDictEqual(response.json(), expected_body) + mock_variant_lookup.assert_called_with( + self.no_access_user, 'phase2_DEL_chr14_4640', '37', sample_type='WGS', ) + + self.login_collaborator() + response = self.client.get(url) + self.assertEqual(response.status_code, 200) mock_variant_lookup.assert_called_with( - self.collaborator_user, 'phase2_DEL_chr14_4640', mock.ANY, genome_version='37', sample_type='WGS', + self.collaborator_user, 'phase2_DEL_chr14_4640', '37', sample_type='WGS', ) - self.assertSetEqual({ - 'F000001_1', 'F000002_2', 'F000003_3', 'F000004_4', 'F000005_5', 'F000006_6', 'F000007_7', 'F000008_8', - 'F000009_9', 'F000010_10', 'F000013_13', - }, {f.guid for f in mock_variant_lookup.call_args.args[2]}) @mock.patch('seqr.views.utils.vlm_utils.VLM_CLIENT_SECRET', 'abc123') @mock.patch('seqr.views.utils.vlm_utils.VLM_CLIENT_ID', MOCK_CLIENT_ID) From 5ec24afac0c34b00cef95f0ca158c4a23a8a75b0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 15 Sep 2025 16:27:36 -0400 Subject: [PATCH 76/89] test sv lookup with no access --- seqr/views/apis/variant_search_api_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 5bbf16854b..0c4cf01bd1 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -1144,6 +1144,7 @@ def test_sv_variant_lookup(self, mock_variant_lookup): self.login_collaborator() response = self.client.get(url) + import pdb; pdb.set_trace() self.assertEqual(response.status_code, 200) mock_variant_lookup.assert_called_with( self.collaborator_user, 'phase2_DEL_chr14_4640', '37', sample_type='WGS', From 70ad47f6253be97d74fdcf681ed7af4f2040abb5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 15 Sep 2025 17:23:08 -0400 Subject: [PATCH 77/89] tets sv looklup with access --- seqr/views/apis/variant_search_api.py | 3 +- seqr/views/apis/variant_search_api_tests.py | 80 +++++++++++++-------- 2 files changed, 52 insertions(+), 31 deletions(-) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 4e0dff6ea1..4673f92bbf 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -590,11 +590,12 @@ def _update_lookup_variant(variant, response, individual_guid_map): add_individual_hpo_details([i for _, i in individual_summary_map.values()]) variant['genotypes'] = {} - variant['lookupFamilyGuids'] = sorted(variant.pop('familyGuids')) + variant['lookupFamilyGuids'] = sorted([guid for guid in variant.pop('familyGuids') if guid in variant['familyGenotypes']]) variant['familyGuids'] = [] for family_guid in variant['lookupFamilyGuids']: for genotype in variant['familyGenotypes'].pop(family_guid): individual_guid = individual_guid_map[(family_guid, genotype['sampleId'])] + genotype = {**genotype, 'individualGuid': individual_guid} if individual_guid in variant['genotypes']: genotype = [variant['genotypes'][individual_guid], genotype] variant['genotypes'][individual_guid] = genotype diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 0c4cf01bd1..e637e90bad 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -999,36 +999,25 @@ def test_variant_lookup(self, mock_variant_lookup): 'liftedFamilyGuids': ['F000014_14'], 'genotypes': { individual_guid: [ - {**sample_genotype, **genotype} for sample_genotype in expected_variant['genotypes'][anon_individual_guid] + {**sample_genotype, **genotype, 'individualGuid': individual_guid} for sample_genotype in expected_variant['genotypes'][anon_individual_guid] ] if isinstance(expected_variant['genotypes'][anon_individual_guid], list) else { - **expected_variant['genotypes'][anon_individual_guid], **genotype, + **expected_variant['genotypes'][anon_individual_guid], **genotype, 'individualGuid': individual_guid, } for individual_guid, anon_individual_guid, genotype in individual_guid_map }, 'genomeVersion': '37', 'variantId': '1-248367227-TC-T', }) + expected_body.update(self._expected_lookup_context( + [PROJECT_GUID, 'R0003_test', 'R0004_non_analyst_project'], + ['F000002_2', 'F000011_11', 'F000014_14'], + [i[0] for i in individual_guid_map] + ['I000019_na21987', 'I000021_na21654']), + ) expected_body.update({ - **{k: {**EXPECTED_SEARCH_RESPONSE[k]} for k in { - 'mmeSubmissionsByGuid', 'variantTagsByGuid', 'variantNotesByGuid', - }}, - **EXPECTED_TRANSCRIPTS_RESPONSE, - 'omimIntervals': {}, - 'savedVariantsByGuid': {k: v for k, v in self.EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if k in ['SV0000002_1248367227_r0390_100']}, - 'variantFunctionalDataByGuid': {}, - 'locusListsByGuid': EXPECTED_SEARCH_CONTEXT_RESPONSE['locusListsByGuid'], - 'projectsByGuid': { - p: {k: mock.ANY for k in PROJECT_TAG_TYPE_FIELDS} - for p in [PROJECT_GUID, 'R0003_test', 'R0004_non_analyst_project'] - }, - 'familiesByGuid': { - f: {k: mock.ANY for k in [*FAMILY_FIELDS, 'individualGuids']} - for f in ['F000002_2', 'F000011_11', 'F000014_14'] - }, - 'individualsByGuid': { - i[0]: {k: mock.ANY for k in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']} - for i in individual_guid_map + [('I000019_na21987',), (('I000021_na21654',))] - }, + k: {**EXPECTED_SEARCH_RESPONSE[k]} for k in {'mmeSubmissionsByGuid', 'variantTagsByGuid', 'variantNotesByGuid'} }) + expected_body['savedVariantsByGuid']= { + k: v for k, v in self.EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if k in ['SV0000002_1248367227_r0390_100'] + } expected_body['genesById']['ENSG00000227232'] = expected_pa_gene expected_body['mmeSubmissionsByGuid']['MS000018_P0004517'] = expected_body['mmeSubmissionsByGuid'].pop('MS000001_na19675') expected_body['savedVariantsByGuid']['SV0000006_1248367227_r0004_non'] = mock.ANY @@ -1057,12 +1046,30 @@ def _expected_lookup_body(self, individuals_by_guid, variants): expected_body['totalSampleCounts'] = self.EXPECTED_SEARCH_RESPONSE['totalSampleCounts'] return expected_body + def _expected_lookup_context(self, project_guids, family_guids, individual_guids): + return { + **EXPECTED_TRANSCRIPTS_RESPONSE, + 'omimIntervals': {}, + 'variantFunctionalDataByGuid': {}, + 'locusListsByGuid': EXPECTED_SEARCH_CONTEXT_RESPONSE['locusListsByGuid'], + 'projectsByGuid': { + p: {k: mock.ANY for k in PROJECT_TAG_TYPE_FIELDS} for p in project_guids + }, + 'familiesByGuid': { + f: {k: mock.ANY for k in [*FAMILY_FIELDS, 'individualGuids']} for f in family_guids + }, + 'individualsByGuid': { + i: {k: mock.ANY for k in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']} for i in individual_guids + }, + } + @mock.patch('seqr.views.apis.variant_search_api.variant_lookup') def test_sv_variant_lookup(self, mock_variant_lookup): - mock_variant_lookup.return_value = deepcopy([ - {**GCNV_LOOKUP_VARIANT, 'familyGenotypes': {k: v for k, v in GCNV_LOOKUP_VARIANT['familyGenotypes'].items() if k != 'F000002_2_x'}} , + lookup_variants = [ + {**GCNV_LOOKUP_VARIANT, 'familyGenotypes': {k: v for k, v in GCNV_LOOKUP_VARIANT['familyGenotypes'].items() if k != 'F000002_2_x'}}, SV_LOOKUP_VARIANT, - ]) + ] + mock_variant_lookup.side_effect = lambda *args, **kwargs: [deepcopy(v) for v in lookup_variants] url = f'{reverse(variant_lookup_handler)}?variantId=phase2_DEL_chr14_4640&genomeVersion=37&sampleType=WGS' self.check_require_login(url) @@ -1142,13 +1149,26 @@ def test_sv_variant_lookup(self, mock_variant_lookup): self.no_access_user, 'phase2_DEL_chr14_4640', '37', sample_type='WGS', ) - self.login_collaborator() + self.login_manager() response = self.client.get(url) - import pdb; pdb.set_trace() self.assertEqual(response.status_code, 200) - mock_variant_lookup.assert_called_with( - self.collaborator_user, 'phase2_DEL_chr14_4640', '37', sample_type='WGS', - ) + expected_gcnv_variant.update({ + 'lookupFamilyGuids': GCNV_VARIANT4['familyGuids'], + 'genotypes': GCNV_VARIANT4['genotypes'], + }) + expected_sv_variant.update({ + 'lookupFamilyGuids': SV_VARIANT4['familyGuids'], + 'genotypes': SV_VARIANT4['genotypes'], + }) + expected_body.update(self._expected_lookup_context( + [PROJECT_GUID, 'R0004_non_analyst_project'], + ['F000002_2', 'F000014_14'], + ['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000018_na21234', 'I000019_na21987', 'I000021_na21654'], + )) + expected_body.update({k: {} for k in {'mmeSubmissionsByGuid', 'variantTagsByGuid', 'variantNotesByGuid'}}) + del expected_body['transcriptsById'] + self.assertDictEqual(response.json(), expected_body) + mock_variant_lookup.assert_called_with(self.manager_user, 'phase2_DEL_chr14_4640', '37', sample_type='WGS') @mock.patch('seqr.views.utils.vlm_utils.VLM_CLIENT_SECRET', 'abc123') @mock.patch('seqr.views.utils.vlm_utils.VLM_CLIENT_ID', MOCK_CLIENT_ID) From 0ab659b4538d4dfab918feb3618e746de393a5f6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 16 Sep 2025 12:14:16 -0400 Subject: [PATCH 78/89] clickhouse only variant lookup --- seqr/utils/search/search_utils_tests.py | 4 ++-- seqr/utils/search/utils.py | 9 +-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 3c968341eb..72329d6ca0 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -540,9 +540,9 @@ def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guid return super()._assert_expected_search_samples(mock_get_variants, omitted_sample_guids, False) def test_variant_lookup(self, *args, **kwargs): - with self.assertRaises(InvalidSearchException) as cm: + with self.assertRaises(ValueError) as cm: super().test_variant_lookup(mock.MagicMock()) - self.assertEqual(str(cm.exception), 'Lookup is disabled') + self.assertEqual(str(cm.exception), 'variant_lookup is disabled without the clickhouse backend') @mock.patch('seqr.utils.search.utils.get_es_variants_for_variant_ids') def test_get_single_variant(self, mock_get_variants_for_ids): diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index d297945b12..487882fb5c 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -50,12 +50,6 @@ class InvalidSearchException(Exception): DATASET_TYPES_LOOKUP[DATASET_TYPE_NO_MITO] = [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_SV_CALLS] -def _raise_search_error(error): - def _wrapped(*args, **kwargs): - raise InvalidSearchException(error) - return _wrapped - - def es_only(func): def _wrapped(*args, **kwargs): if not es_backend_enabled(): @@ -152,9 +146,8 @@ def _get_clickhouse_variant_by_id(parsed_variant_id, variant_id, samples, genome ) +@clickhouse_only def variant_lookup(user, variant_id, genome_version, sample_type=None): - backend_specific_call(_raise_search_error('Lookup is disabled'), lambda: None)() - cache_key = f'variant_lookup_results__{variant_id}__{genome_version}' variants = safe_redis_get_json(cache_key) if variants: From ce543ef879477615bb1ff7d1b7b4847b63d89e2a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 16 Sep 2025 12:24:11 -0400 Subject: [PATCH 79/89] add dataset es only --- seqr/utils/search/add_data_utils.py | 45 ++-------------------------- seqr/views/apis/dataset_api.py | 41 +++++++++++++++++++++++-- seqr/views/apis/dataset_api_tests.py | 10 +++---- 3 files changed, 45 insertions(+), 51 deletions(-) diff --git a/seqr/utils/search/add_data_utils.py b/seqr/utils/search/add_data_utils.py index 538a182499..8d82255083 100644 --- a/seqr/utils/search/add_data_utils.py +++ b/seqr/utils/search/add_data_utils.py @@ -8,10 +8,7 @@ from seqr.utils.communication_utils import send_project_notification from seqr.utils.file_utils import does_file_exist from seqr.utils.logging_utils import SeqrLogger -from seqr.utils.search.utils import backend_specific_call -from seqr.utils.search.elasticsearch.es_utils import validate_es_index_metadata_and_get_samples from seqr.views.utils.airtable_utils import AirtableSession, ANVIL_REQUEST_TRACKING_TABLE -from seqr.views.utils.dataset_utils import match_and_update_search_samples, load_mapping_file from seqr.views.utils.export_utils import write_multiple_files from seqr.views.utils.pedigree_info_utils import JsonConstants from settings import SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, BASE_URL, ANVIL_UI_URL, \ @@ -20,45 +17,7 @@ logger = SeqrLogger(__name__) -def _no_es_backend_error(*args, **kwargs): - raise ValueError('Adding samples is disabled without the elasticsearch backend') - - -def add_new_es_search_samples(request_json, project, user, notify=False, expected_families=None): - dataset_type = request_json.get('datasetType') - if dataset_type not in Sample.DATASET_TYPE_LOOKUP: - raise ValueError(f'Invalid dataset type "{dataset_type}"') - - sample_ids, sample_type, sample_data = backend_specific_call( - validate_es_index_metadata_and_get_samples, - _no_es_backend_error, - )(request_json, project) - if not sample_ids: - raise ValueError('No samples found. Make sure the specified caller type is correct') - - sample_id_to_individual_id_mapping = load_mapping_file( - request_json['mappingFilePath'], user) if request_json.get('mappingFilePath') else {} - ignore_extra_samples = request_json.get('ignoreExtraSamplesInCallset') - sample_project_tuples = [(sample_id, project.name) for sample_id in sample_ids] - new_samples, updated_samples, inactivated_sample_guids, updated_family_guids = match_and_update_search_samples( - projects=[project], - user=user, - sample_project_tuples=sample_project_tuples, - sample_data=sample_data, - sample_type=sample_type, - dataset_type=dataset_type, - expected_families=expected_families, - sample_id_to_individual_id_mapping=sample_id_to_individual_id_mapping, - raise_unmatched_error_template=None if ignore_extra_samples else 'Matches not found for sample ids: {sample_ids}. Uploading a mapping file for these samples, or select the "Ignore extra samples in callset" checkbox to ignore.' - ) - - if notify: - _basic_notify_search_data_loaded(project, dataset_type, sample_type, new_samples.values_list('sample_id')) - - return inactivated_sample_guids, updated_family_guids, updated_samples - - -def _basic_notify_search_data_loaded(project, dataset_type, sample_type, new_samples, email_template=None, slack_channel=None, include_slack_detail=False): +def basic_notify_search_data_loaded(project, dataset_type, sample_type, new_samples, email_template=None, slack_channel=None, include_slack_detail=False): msg_dataset_type = '' if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS else f' {dataset_type}' num_new_samples = len(new_samples) sample_summary = f'{num_new_samples} new {sample_type}{msg_dataset_type} samples' @@ -86,7 +45,7 @@ def notify_search_data_loaded(project, is_internal, dataset_type, sample_type, n 'Let us know if you have any questions.', ]) - _basic_notify_search_data_loaded( + basic_notify_search_data_loaded( project, dataset_type, sample_type, new_samples, email_template=email_template, slack_channel=SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL if is_internal else SEQR_SLACK_ANVIL_DATA_LOADING_CHANNEL, include_slack_detail=is_internal, diff --git a/seqr/views/apis/dataset_api.py b/seqr/views/apis/dataset_api.py index 078ccb0dda..e0dfa35a1d 100644 --- a/seqr/views/apis/dataset_api.py +++ b/seqr/views/apis/dataset_api.py @@ -1,13 +1,17 @@ import json -from seqr.utils.search.add_data_utils import add_new_es_search_samples -from seqr.models import Individual, Family +from seqr.utils.search.add_data_utils import basic_notify_search_data_loaded +from seqr.utils.search.elasticsearch.es_utils import validate_es_index_metadata_and_get_samples +from seqr.utils.search.utils import es_only +from seqr.models import Individual, Family, Sample +from seqr.views.utils.dataset_utils import match_and_update_search_samples, load_mapping_file from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.orm_to_json_utils import get_json_for_samples from seqr.views.utils.permissions_utils import get_project_and_check_permissions, data_manager_required @data_manager_required +@es_only def add_variants_dataset_handler(request, project_guid): """Create or update samples for the given variant dataset @@ -31,7 +35,7 @@ def add_variants_dataset_handler(request, project_guid): request_json = json.loads(request.body) try: - inactivated_sample_guids, updated_family_guids, updated_samples = add_new_es_search_samples( + inactivated_sample_guids, updated_family_guids, updated_samples = _add_new_es_search_samples( request_json, project, request.user, notify=True) except ValueError as e: return create_json_response({'errors': [str(e)]}, status=400) @@ -43,6 +47,37 @@ def add_variants_dataset_handler(request, project_guid): return create_json_response(response_json) +def _add_new_es_search_samples(request_json, project, user, notify=False, expected_families=None): + dataset_type = request_json.get('datasetType') + if dataset_type not in Sample.DATASET_TYPE_LOOKUP: + raise ValueError(f'Invalid dataset type "{dataset_type}"') + + sample_ids, sample_type, sample_data = validate_es_index_metadata_and_get_samples(request_json, project) + if not sample_ids: + raise ValueError('No samples found. Make sure the specified caller type is correct') + + sample_id_to_individual_id_mapping = load_mapping_file( + request_json['mappingFilePath'], user) if request_json.get('mappingFilePath') else {} + ignore_extra_samples = request_json.get('ignoreExtraSamplesInCallset') + sample_project_tuples = [(sample_id, project.name) for sample_id in sample_ids] + new_samples, updated_samples, inactivated_sample_guids, updated_family_guids = match_and_update_search_samples( + projects=[project], + user=user, + sample_project_tuples=sample_project_tuples, + sample_data=sample_data, + sample_type=sample_type, + dataset_type=dataset_type, + expected_families=expected_families, + sample_id_to_individual_id_mapping=sample_id_to_individual_id_mapping, + raise_unmatched_error_template=None if ignore_extra_samples else 'Matches not found for sample ids: {sample_ids}. Uploading a mapping file for these samples, or select the "Ignore extra samples in callset" checkbox to ignore.' + ) + + if notify: + basic_notify_search_data_loaded(project, dataset_type, sample_type, new_samples.values_list('sample_id')) + + return inactivated_sample_guids, updated_family_guids, updated_samples + + def _get_samples_json(updated_samples, inactivated_sample_guids, project_guid): updated_sample_json = get_json_for_samples(updated_samples, project_guid=project_guid) sample_response = {sample_guid: {'isActive': False} for sample_guid in inactivated_sample_guids} diff --git a/seqr/views/apis/dataset_api_tests.py b/seqr/views/apis/dataset_api_tests.py index 1428c80f4d..091f3abc08 100644 --- a/seqr/views/apis/dataset_api_tests.py +++ b/seqr/views/apis/dataset_api_tests.py @@ -284,6 +284,9 @@ def test_add_variants_dataset_errors(self): url = reverse(add_variants_dataset_handler, args=[PROJECT_GUID]) self.check_data_manager_login(url) + self._assert_expected_add_dataset_errors(url) + + def _assert_expected_add_dataset_errors(self, url): response = self.client.post(url, content_type='application/json', data=json.dumps({})) self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'errors': ['Invalid dataset type "None"']}) @@ -293,9 +296,6 @@ def test_add_variants_dataset_errors(self): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'errors': ['Invalid dataset type "NOT_A_TYPE"']}) - self._assert_expected_add_dataset_errors(url) - - def _assert_expected_add_dataset_errors(self, url): response = self.client.post(url, content_type='application/json', data=json.dumps({'datasetType': 'SV'})) self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'errors': ['request must contain field: "elasticsearchIndex"']}) @@ -463,8 +463,8 @@ class AnvilDatasetAPITest(AnvilAuthenticationTestCase, DatasetAPITest): def _assert_expected_add_dataset_errors(self, url): response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['errors'][0], 'Adding samples is disabled without the elasticsearch backend') + self.assertEqual(response.status_code, 500) + self.assertEqual(response.json()['error'], 'add_variants_dataset_handler is disabled without the elasticsearch backend') def test_add_variants_dataset(self, *args): # Adding dataset is always disabled when ES is disabled, which is tested in test_add_variants_dataset_errors From 5f42c58ceffccd3ef3cf479251d68ebe397b8d23 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 16 Sep 2025 12:39:55 -0400 Subject: [PATCH 80/89] todos --- seqr/utils/search/utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 25bb6bad9e..ca50f8fed9 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -254,7 +254,7 @@ def _get_cached_search_results(search_model, sort=None, family_guid=None): cache_key = _get_search_cache_key(search_model, sort=sort) results = safe_redis_get_json(cache_key) if not results: - results = backend_specific_call( + results = backend_specific_call( # TODO lambda *args: None, _process_clickhouse_unsorted_cached_results, )(cache_key, sort, family_guid) return results or {} @@ -280,12 +280,12 @@ def query_variants(search_model, sort=XPOS_SORT_KEY, skip_genotype_filter=False, genome_version = _get_search_genome_version(search_model.families.all()) loaded_results = previous_search_results.get('all_results') or [] if len(loaded_results) >= end_index: - results_page = backend_specific_call( + results_page = backend_specific_call( # TODO lambda results, genome_version: results, format_clickhouse_results, )(loaded_results[start_index:end_index], genome_version) return results_page, total_results - previously_loaded_results = backend_specific_call( + previously_loaded_results = backend_specific_call( # TODO process_es_previously_loaded_results, lambda *args: None, # Other backends need no additional parsing )(previous_search_results, start_index, end_index) @@ -390,7 +390,7 @@ def get_variant_query_gene_counts(search_model, user): if len(previous_search_results.get('all_results', [])) == previous_search_results.get('total_results'): return _get_gene_aggs_for_cached_variants(previous_search_results) - previously_loaded_results = backend_specific_call( + previously_loaded_results = backend_specific_call( # TODO process_es_previously_loaded_gene_aggs, lambda *args: None, # Other backends need no additional parsing )(previous_search_results) @@ -399,7 +399,7 @@ def get_variant_query_gene_counts(search_model, user): genome_version = _get_search_genome_version(search_model.families.all()) gene_counts, _ = _query_variants(search_model, user, previous_search_results, genome_version, gene_agg=True) - gene_counts = backend_specific_call( + gene_counts = backend_specific_call( # TODO lambda *args: None, _get_gene_aggs_for_cached_variants, )(previous_search_results) or gene_counts return gene_counts @@ -408,13 +408,13 @@ def get_variant_query_gene_counts(search_model, user): def _get_gene_aggs_for_cached_variants(previous_search_results): gene_aggs = defaultdict(lambda: {'total': 0, 'families': defaultdict(int)}) # ES caches compound hets separately from main results, other backends cache everything together - flattened_variants = backend_specific_call( + flattened_variants = backend_specific_call( # TODO lambda results: results, lambda results: [v for variants in results for v in (variants if isinstance(variants, list) else [variants])], )(previous_search_results['all_results']) for var in flattened_variants: # ES only reports breakdown for main transcript gene only, other backends report for all genes - gene_ids = backend_specific_call( + gene_ids = backend_specific_call( # TODO lambda variant: next(( [gene_id] for gene_id, transcripts in variant['transcripts'].items() if any(t['transcriptId'] == var['mainTranscriptId'] for t in transcripts) From bf14c6a60f1bd2494034aa0206025455eb867900 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 16 Sep 2025 15:04:17 -0400 Subject: [PATCH 81/89] remove unnecessary parsed locus processing --- clickhouse_search/managers.py | 85 ++++++++++++--------- clickhouse_search/search.py | 21 ++--- seqr/utils/search/elasticsearch/es_utils.py | 4 +- seqr/utils/search/utils.py | 70 +++++------------ 4 files changed, 83 insertions(+), 97 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 7cf4477d1b..983113b8f1 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -205,12 +205,11 @@ def _get_join_query_values(self, query, alias, conditional_selects): **{f'{alias}_{field}': value for field, value in query_select.items()}, ) - def search(self, parsed_locus=None, **kwargs): - parsed_locus = parsed_locus or {} - results = self.filter_variant_ids(**parsed_locus) + def search(self, **kwargs): + results = self.filter_variant_ids(**kwargs) results = self._filter_frequency(results, **kwargs) results = self._filter_in_silico(results, **kwargs) - results = self.filter_annotations(results, **parsed_locus, **kwargs) + results = self.filter_annotations(results, **kwargs, **kwargs) return results def result_values(self, skip_entry_fields=False): @@ -323,11 +322,11 @@ def search_compound_hets(self, primary_q, secondary_q): **{primary_gene_field: F(secondary_gene_field)} ).exclude(primary_variantId=F('secondary_variantId')) - def filter_variant_ids(self, variant_ids=None, rs_ids=None, **kwargs): + def filter_variant_ids(self, parsed_variant_ids=None, rs_ids=None, **kwargs): results = self - if variant_ids: + if parsed_variant_ids: results = results.filter( - variant_id__in=[f'{chrom}-{pos}-{ref}-{alt}' for chrom, pos, ref, alt in variant_ids] + variant_id__in=[f'{chrom}-{pos}-{ref}-{alt}' for chrom, pos, ref, alt in parsed_variant_ids] ) if rs_ids: @@ -413,7 +412,7 @@ def _get_in_silico_score_q(self, score, value): except ValueError: return Q(**{score_column: value}) - def filter_annotations(self, results, annotations=None, pathogenicity=None, exclude=None, gene_ids=None, intervals=None, exclude_intervals=False, padded_interval_end=None, **kwargs): + def filter_annotations(self, results, annotations=None, pathogenicity=None, exclude=None, genes=None, intervals=None, exclude_locations=False, padded_interval_end=None, **kwargs): transcript_field = self.transcript_field if self.model.GENOTYPE_OVERRIDE_FIELDS: results = results.annotate(**{ @@ -422,11 +421,11 @@ def filter_annotations(self, results, annotations=None, pathogenicity=None, excl }]), }) transcript_field = self.GENOTYPE_GENE_CONSEQUENCE_FIELD - interval_qs = [self._interval_query(*interval) for interval in intervals or []] - if gene_ids: + interval_qs = [self._interval_query(**interval) for interval in intervals or []] + if genes: results = results.annotate(**{ self.GENE_CONSEQUENCE_FIELD: ArrayFilter(transcript_field, conditions=[{ - 'geneId': (gene_ids, 'has({value}, {field})'), + 'geneId': (sorted(genes.keys()), 'has({value}, {field})'), }]), }) gene_q = Q(gene_consequences__not_empty=True) @@ -440,7 +439,7 @@ def filter_annotations(self, results, annotations=None, pathogenicity=None, excl interval_q = interval_qs[0] for q in interval_qs[1:]: interval_q |= q - filter_func = results.exclude if exclude_intervals else results.filter + filter_func = results.exclude if exclude_locations else results.filter results = filter_func(interval_q) filter_qs, transcript_filters = self._parse_annotation_filters(annotations, pathogenicity) if (annotations or pathogenicity) else ([], []) @@ -463,7 +462,7 @@ def filter_annotations(self, results, annotations=None, pathogenicity=None, excl filter_q = Q(passes_annotation=True) if transcript_filters: - consequence_field = self.GENE_CONSEQUENCE_FIELD if gene_ids else transcript_field + consequence_field = self.GENE_CONSEQUENCE_FIELD if genes else transcript_field results = results.annotate(**{ self.FILTERED_CONSEQUENCE_FIELD: ArrayFilter(consequence_field, conditions=transcript_filters), }) @@ -475,7 +474,11 @@ def filter_annotations(self, results, annotations=None, pathogenicity=None, excl return results.filter(filter_q) - def _interval_query(self, chrom, start, end): + def _interval_query(self, chrom, start, end, offset=None, **kwargs): + if offset: + offset_pos = int((end - start) * offset) + start = max(start - offset_pos, MIN_POS) + end = min(end + offset_pos, MAX_POS) q = Q(xpos__range=(get_xpos(chrom, start), get_xpos(chrom, end))) if hasattr(self.model, 'end_chrom'): q |= Q(end_chrom__isnull=True, chrom=chrom, end__range=(start, end)) @@ -683,8 +686,8 @@ def genotype_fields(self): def clinvar_model(self): return self.model.clinvar_join.rel.related_model - def search(self, sample_data, parsed_locus=None, freqs=None, annotations=None, **kwargs): - entries = self.filter_locus(**(parsed_locus or {})) + def search(self, sample_data, freqs=None, annotations=None, **kwargs): + entries = self.filter_locus(**kwargs) entries = self._join_annotations(entries) @@ -1043,45 +1046,45 @@ def _family_passes_expression(pass_field): mapped_expression='x.1', output_field=models.ArrayField(models.StringField()), ) - def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=None, gene_ids=None, variant_ids=None, require_gene_filter=False, **kwargs): + def filter_locus(self, exclude_locations=False, intervals=None, genes=None, parsed_variant_ids=None, require_gene_filter=False, **kwargs): entries = self - if variant_ids: + if parsed_variant_ids: # although technically redundant, the interval query is applied to the entries table before join and reduces the join size, # while the full variant_id filter is applied to the annotation table after the join - intervals = [(chrom, pos, pos) for chrom, pos, _, _ in variant_ids] + intervals = [{'chrom': chrom, 'start': pos, 'end': pos} for chrom, pos, _, _ in parsed_variant_ids] + should_filter_interval = False if 'cn' in self.call_fields: # SV interval filtering occurs after joining on annotations to correctly incorporate end position - if exclude_intervals: + if exclude_locations: intervals = None else: - if gene_ids and 'geneIds' in self.call_fields: + if genes and 'geneIds' in self.call_fields: entries = entries.filter(calls__array_all={'OR': [ - {'geneIds': (gene_ids, 'hasAny({value}, {field})')}, + {'geneIds': (list(genes.keys()), 'hasAny({value}, {field})')}, {'gt': (None, 'isNull({field})')}, ]}) - intervals = list((gene_intervals or {}).values()) + (intervals or []) - gene_intervals = None - chromosomes = {chrom for chrom, _, _ in (intervals or [])} - intervals = [(chrom, MIN_POS, MAX_POS) for chrom in chromosomes] + should_filter_interval = True + chromosomes = {i['chrom'] for i in (intervals or [])} + intervals = [{'chrom': chrom, 'start': MIN_POS, 'end': MAX_POS} for chrom in chromosomes] - if not (gene_intervals or intervals): + if not (genes or intervals): return entries locus_q = None - if gene_intervals: + if genes: if hasattr(self.model, 'is_annotated_in_any_gene') and not intervals: entries = entries.filter(is_annotated_in_any_gene=Value(True)) - should_filter_interval = (not hasattr(self.model, 'geneId_ids')) or exclude_intervals or len(gene_intervals) < self.model.MAX_XPOS_FILTER_INTERVALS + should_filter_interval |= (not hasattr(self.model, 'geneId_ids')) or exclude_locations or len(genes) < self.model.MAX_XPOS_FILTER_INTERVALS if should_filter_interval: - intervals = list((gene_intervals or {}).values()) + (intervals or []) + intervals = self._format_gene_intervals(genes) + (intervals or []) if require_gene_filter or (not should_filter_interval): - locus_q = Q(geneId_ids__bitmap_has_any=list(gene_intervals.keys())) + locus_q = Q(geneId_ids__bitmap_has_any=[gene['id'] for gene in genes.values()]) if intervals: - interval_q = self._interval_query(*intervals[0]) + interval_q = self._interval_query(**intervals[0]) for interval in intervals[1:]: - interval_q |= self._interval_query(*interval) + interval_q |= self._interval_query(**interval) if locus_q is None: locus_q = interval_q elif require_gene_filter: @@ -1089,15 +1092,25 @@ def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=N else: locus_q |= interval_q - filter_func = entries.exclude if exclude_intervals else entries.filter + filter_func = entries.exclude if exclude_locations else entries.filter return filter_func(locus_q) + def _format_gene_intervals(self, genes): + genome_version = self.annotations_model.ANNOTATION_CONSTANTS['genomeVersion'] + return [ + {field: gene[f'{field}Grch{genome_version}'] for field in ['chrom', 'start', 'end']} for gene in genes.values() + ] + def search_padded_interval(self, chrom, pos, padding): - interval_q = self._interval_query(chrom, max(pos - padding, MIN_POS), min(pos + padding, MAX_POS)) + interval_q = self._interval_query(chrom, start=max(pos - padding, MIN_POS), end=min(pos + padding, MAX_POS)) return self.filter(interval_q).result_values() @staticmethod - def _interval_query(chrom, start, end): + def _interval_query(chrom, start, end, offset=None, **kwargs): + if offset: + offset_pos = int((end - start) * offset) + start = max(start - offset_pos, MIN_POS) + end = min(end + offset_pos, MAX_POS) return Q(xpos__range=(get_xpos(chrom, start), get_xpos(chrom, end))) def _filter_seqr_frequency(self, entries, ac=None, hh=None, **kwargs): diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 315a45fded..fd8106fea8 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -49,7 +49,7 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno dataset_results += _evaluate_results(result_q) if has_comp_het: comp_het_sample_data = sample_data - if is_multi_project and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and _is_x_chrom_only(**search): + if is_multi_project and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and _is_x_chrom_only(genome_version, **search): comp_het_sample_data = _no_affected_male_families(sample_data, user) result_q = _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, comp_het_sample_data, skip_individual_guid=is_multi_project, **search) dataset_results += _evaluate_results(result_q, is_comp_het=True) @@ -386,10 +386,10 @@ def _no_affected_male_families(sample_data, user): } -def _is_x_chrom_only(parsed_locus=None, **kwargs): - parsed_locus = parsed_locus or {} - all_intervals = list((parsed_locus.get('gene_intervals') or {}).values()) + (parsed_locus.get('intervals') or []) - return bool(all_intervals) and all('X' in interval[0] for interval in all_intervals) +def _is_x_chrom_only(genome_version, genes=None, intervals=None, **kwargs): + if not (genes or intervals): + return False + return bool('X' in gene[f'chromGrch{genome_version}'] for gene in (genes or {}).values()) and all('X' in interval['chrom'] for interval in (intervals or [])) OMIM_SORT = 'in_omim' @@ -537,7 +537,7 @@ def _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples=No keys = KEY_LOOKUP_CLASS_MAP[genome_version][data_type].objects.filter(variant_id=variant_id).values_list('key', flat=True) entries = entry_cls.objects.filter(key__in=keys) else: - entries = entry_cls.objects.filter_locus(variant_ids=[variant_id]) + entries = entry_cls.objects.filter_locus(parsed_variant_ids=[variant_id]) entries = entries.result_values(sample_data) results = annotations_cls.objects.subquery_join(entries) @@ -545,7 +545,7 @@ def _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples=No # Handles annotation for genotype overrides results = results.filter_annotations(results) else: - results = results.filter_variant_ids(variant_ids=[variant_id]) + results = results.filter_variant_ids(parsed_variant_ids=[variant_id]) variant = results.result_values().first() if variant: @@ -602,7 +602,7 @@ def _add_liftover_genotypes(variant, data_type, variant_id): ).values_list('key', flat=True) if not keys: return - lifted_entries = lifted_entry_cls.objects.filter_locus(variant_ids=[lifted_id]).filter(key=keys[0]) + lifted_entries = lifted_entry_cls.objects.filter_locus(parsed_variant_ids=[lifted_id]).filter(key=keys[0]) lifted_entry_data = lifted_entries.values('key').annotate( familyGenotypes=GroupArrayArray(lifted_entry_cls.objects.genotype_expression()) ) @@ -664,8 +664,9 @@ def get_clickhouse_keys_for_gene(gene_id, genome_version, dataset_type, keys): def get_all_clickhouse_keys_for_gene(gene_model, genome_version): entry_cls = ENTRY_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS] - return list(entry_cls.objects.filter_locus(require_gene_filter=True, gene_intervals={ - gene_model.id: [getattr(gene_model, f'{field}_grch{genome_version}') for field in ['chrom', 'start', 'end']] + return list(entry_cls.objects.filter_locus(require_gene_filter=True, genes={gene_model.gene_id: { + 'id': gene_model.id, + **{f'{field}Grch{genome_version}': getattr(gene_model, f'{field}_grch{genome_version}') for field in ['chrom', 'start', 'end']}}, }).values_list('key', flat=True).distinct()) diff --git a/seqr/utils/search/elasticsearch/es_utils.py b/seqr/utils/search/elasticsearch/es_utils.py index 94ffead964..b5386c122f 100644 --- a/seqr/utils/search/elasticsearch/es_utils.py +++ b/seqr/utils/search/elasticsearch/es_utils.py @@ -290,8 +290,8 @@ def get_es_variants(samples, search, user, previous_search_results, genome_versi in_silico=search.get('in_silico'), quality_filter=search.get('qualityFilter'), custom_query=search.get('customQuery'), locus=search.get('locus'), skip_genotype_filter=skip_genotype_filter, dataset_type=search.get('dataset_type'), secondary_dataset_type=search.get('secondary_dataset_type'), - **search.get('parsed_locus') - + genes=search.get('genes'), intervals=search.get('intervals'), rs_ids=search.get('rs_ids'), + variant_ids=search.get('variant_ids'), exclude_locations=search.get('exclude_locations'), ) return es_search.search(page=page, num_results=num_results) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 85b7c3c6e4..0400a9e809 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -281,18 +281,19 @@ def _query_variants(search_model, user, previous_search_results, genome_version, if locus and exclude_locations: raise InvalidSearchException('Cannot specify both Location and Excluded Genes/Intervals') - rs_ids = None variant_ids = None - parsed_variant_ids = None + parsed_search = {**search} genes, intervals, invalid_items = parse_locus_list_items(locus or exclude, genome_version=genome_version, additional_model_fields=['id']) if invalid_items: raise InvalidSearchException('Invalid genes/intervals: {}'.format(', '.join(invalid_items))) + parsed_search.update({'genes': genes, 'intervals': intervals, 'exclude_locations': exclude_locations}) if not (genes or intervals): rs_ids, variant_ids, parsed_variant_ids, invalid_items = _parse_variant_items(locus) if invalid_items: raise InvalidSearchException('Invalid variants: {}'.format(', '.join(invalid_items))) if rs_ids and variant_ids: raise InvalidSearchException('Invalid variant notation: found both variant IDs and rsIDs') + parsed_search.update({'rs_ids': rs_ids, 'variant_ids': variant_ids, 'parsed_variant_ids': parsed_variant_ids}) if variant_ids: num_results = len(variant_ids) @@ -303,18 +304,11 @@ def _query_variants(search_model, user, previous_search_results, genome_version, if duplicates: raise InvalidSearchException(f'ClinVar pathogenicity {", ".join(sorted(duplicates))} is both included and excluded') - parsed_search = { - 'parsed_locus': backend_specific_call( - lambda genome_version, **kwargs: kwargs, _parse_locus_gene_intervals, - )(genome_version, genes=genes, intervals=intervals, rs_ids=rs_ids, variant_ids=variant_ids, - parsed_variant_ids=parsed_variant_ids, exclude_locations=exclude_locations), - } - parsed_search.update(search) for annotation_key in ['annotations', 'annotations_secondary']: if parsed_search.get(annotation_key): parsed_search[annotation_key] = {k: v for k, v in parsed_search[annotation_key].items() if v} - dataset_type, secondary_dataset_type, lookup_dataset_type = _search_dataset_type(parsed_search) + dataset_type, secondary_dataset_type, lookup_dataset_type = _search_dataset_type(parsed_search, genome_version) parsed_search.update({'dataset_type': dataset_type, 'secondary_dataset_type': secondary_dataset_type}) search_dataset_type = None if dataset_type and dataset_type != ALL_DATA_TYPES: @@ -436,17 +430,18 @@ def _validate_sort(sort, families): raise InvalidSearchException('Phenotype sort is only supported for single-family search.') -def _search_dataset_type(search): - locus = search['parsed_locus'] - parsed_variant_ids = locus.get('parsed_variant_ids', locus['variant_ids']) - rsids = locus.get('rs_ids') +def _search_dataset_type(search, genome_version): + parsed_variant_ids = search.get('parsed_variant_ids') + rsids = search.get('rs_ids') if parsed_variant_ids or rsids: lookup_dataset_type = Sample.DATASET_TYPE_VARIANT_CALLS if rsids else _variant_ids_dataset_type(parsed_variant_ids) return Sample.DATASET_TYPE_VARIANT_CALLS, None, lookup_dataset_type - intervals = (locus['intervals'] or (locus.get('gene_intervals') or {}).values()) if 'exclude_intervals' in locus and not locus['exclude_intervals'] else None - dataset_type = _annotation_dataset_type(search.get('annotations'), intervals, pathogenicity=search.get('pathogenicity')) - secondary_dataset_type = _annotation_dataset_type(search['annotations_secondary'], intervals) if search.get('annotations_secondary') else None + chroms = [gene[f'chromGrch{genome_version}'] for gene in (search.get('genes') or {}).values()] + [ + interval['chrom'] for interval in (search.get('intervals') or []) + ] + dataset_type = _annotation_dataset_type(search.get('annotations'), chroms, pathogenicity=search.get('pathogenicity')) + secondary_dataset_type = _annotation_dataset_type(search['annotations_secondary'], chroms) if search.get('annotations_secondary') else None return dataset_type, secondary_dataset_type, None @@ -456,19 +451,19 @@ def _variant_ids_dataset_type(all_variant_ids): any_sv = len(variant_ids) < len(all_variant_ids) if len(variant_ids) == 0: return Sample.DATASET_TYPE_SV_CALLS - return _chromosome_filter_dataset_type(variant_ids, any_sv) + return _chromosome_filter_dataset_type([vid[0] for vid in variant_ids], any_sv) -def _chromosome_filter_dataset_type(loci, any_sv): - has_mito = [locus[0] for locus in loci if locus[0].replace('chr', '').startswith('M')] - if len(has_mito) == len(loci): +def _chromosome_filter_dataset_type(chroms, any_sv): + has_mito = [chrom for chrom in chroms if chrom.replace('chr', '').startswith('M')] + if len(has_mito) == len(chroms): return Sample.DATASET_TYPE_MITO_CALLS elif not has_mito: return DATASET_TYPE_NO_MITO if any_sv else DATASET_TYPE_SNP_INDEL_ONLY return ALL_DATA_TYPES if any_sv else Sample.DATASET_TYPE_VARIANT_CALLS -def _annotation_dataset_type(annotations, intervals, pathogenicity=None): - if not (annotations or intervals): +def _annotation_dataset_type(annotations, chroms, pathogenicity=None): + if not (annotations or chroms): return Sample.DATASET_TYPE_VARIANT_CALLS if pathogenicity else None annotation_types = set((annotations or {}).keys()) @@ -476,8 +471,8 @@ def _annotation_dataset_type(annotations, intervals, pathogenicity=None): return Sample.DATASET_TYPE_SV_CALLS no_svs = (annotations and annotation_types.isdisjoint(SV_ANNOTATION_TYPES)) - if intervals: - return _chromosome_filter_dataset_type(intervals, any_sv=not no_svs) + if chroms: + return _chromosome_filter_dataset_type(chroms, any_sv=not no_svs) elif no_svs: return Sample.DATASET_TYPE_VARIANT_CALLS return ALL_DATA_TYPES @@ -512,7 +507,7 @@ def _parse_inheritance(search, samples): def _validate_search(search, samples, previous_search_results): has_comp_het_search = search.get('inheritance_mode') in {RECESSIVE, COMPOUND_HET} and not previous_search_results.get('grouped_results') - has_location_filter = any(search['parsed_locus'].get(field) for field in ['genes', 'gene_ids', 'intervals', 'variant_ids']) + has_location_filter = any(search.get(field) for field in ['genes', 'intervals', 'variant_ids']) if has_comp_het_search: if not search.get('annotations'): raise InvalidSearchException('Annotations must be specified to search for compound heterozygous variants') @@ -576,29 +571,6 @@ def _filter_inheritance_family_samples(samples, inheritance_filter): ] -def _parse_locus_gene_intervals(genome_version, genes=None, intervals=None, rs_ids=None, parsed_variant_ids=None, exclude_locations=False, **kwargs): - intervals = [_format_interval(**interval) for interval in intervals or []] - gene_intervals = gene_ids = None - if genes: - gene_intervals = { - gene['id']: [gene[f'{field}Grch{genome_version}'] for field in ['chrom', 'start', 'end']] for gene in genes.values() - } - if exclude_locations: - intervals += sorted(gene_intervals.values()) - gene_intervals = None - else: - gene_ids = sorted(genes.keys()) - - return { - 'intervals': intervals or None, - 'gene_intervals': gene_intervals, - 'exclude_intervals': exclude_locations, - 'gene_ids': gene_ids, - 'variant_ids': parsed_variant_ids, - 'rs_ids': rs_ids, - } - - def _format_interval(chrom=None, start=None, end=None, offset=None, **kwargs): if offset: offset_pos = int((end - start) * offset) From d0bce14e94c4111c14f8c51835426fcc4b10ede8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 16 Sep 2025 16:20:16 -0400 Subject: [PATCH 82/89] fix clickhouse errors --- clickhouse_search/managers.py | 27 +++++++++++++++++++-------- seqr/utils/search/utils.py | 2 +- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 983113b8f1..7351b83809 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -54,6 +54,11 @@ def _seqr_pop_fields(self, seqr_populations): seqr_pop_fields += [f"'{sub_fields['hom']}_{sample_type}'" for sample_type in sample_types] return seqr_pop_fields + def _format_gene_intervals(self, genes): + return [ + {field: gene[f'{field}Grch{self.genome_version}'] for field in ['chrom', 'start', 'end']} for gene in genes.values() + ] + class AnnotationsQuerySet(SearchQuerySet): @@ -154,6 +159,10 @@ def clinvar_model(self): def clinvar_field_prefix(self): return f'{self.entry_field}__clinvar_join' + @property + def genome_version(self): + return self.model.ANNOTATION_CONSTANTS['genomeVersion'] + def subquery_join(self, subquery, join_key='key'): join_field = next(field for field in subquery.model._meta.fields if field.name == join_key) @@ -209,7 +218,7 @@ def search(self, **kwargs): results = self.filter_variant_ids(**kwargs) results = self._filter_frequency(results, **kwargs) results = self._filter_in_silico(results, **kwargs) - results = self.filter_annotations(results, **kwargs, **kwargs) + results = self.filter_annotations(results, **kwargs) return results def result_values(self, skip_entry_fields=False): @@ -422,6 +431,9 @@ def filter_annotations(self, results, annotations=None, pathogenicity=None, excl }) transcript_field = self.GENOTYPE_GENE_CONSEQUENCE_FIELD interval_qs = [self._interval_query(**interval) for interval in intervals or []] + if exclude_locations and genes: + interval_qs += [self._interval_query(**interval) for interval in self._format_gene_intervals(genes)] + genes = None if genes: results = results.annotate(**{ self.GENE_CONSEQUENCE_FIELD: ArrayFilter(transcript_field, conditions=[{ @@ -686,6 +698,10 @@ def genotype_fields(self): def clinvar_model(self): return self.model.clinvar_join.rel.related_model + @property + def genome_version(self): + return self.annotations_model.ANNOTATION_CONSTANTS['genomeVersion'] + def search(self, sample_data, freqs=None, annotations=None, **kwargs): entries = self.filter_locus(**kwargs) @@ -1059,13 +1075,14 @@ def filter_locus(self, exclude_locations=False, intervals=None, genes=None, pars if exclude_locations: intervals = None else: + chromosomes = {i['chrom'] for i in (intervals or [])} if genes and 'geneIds' in self.call_fields: entries = entries.filter(calls__array_all={'OR': [ {'geneIds': (list(genes.keys()), 'hasAny({value}, {field})')}, {'gt': (None, 'isNull({field})')}, ]}) + chromosomes.update({gene[f'chromGrch{self.genome_version}'] for gene in genes.values()}) should_filter_interval = True - chromosomes = {i['chrom'] for i in (intervals or [])} intervals = [{'chrom': chrom, 'start': MIN_POS, 'end': MAX_POS} for chrom in chromosomes] if not (genes or intervals): @@ -1095,12 +1112,6 @@ def filter_locus(self, exclude_locations=False, intervals=None, genes=None, pars filter_func = entries.exclude if exclude_locations else entries.filter return filter_func(locus_q) - def _format_gene_intervals(self, genes): - genome_version = self.annotations_model.ANNOTATION_CONSTANTS['genomeVersion'] - return [ - {field: gene[f'{field}Grch{genome_version}'] for field in ['chrom', 'start', 'end']} for gene in genes.values() - ] - def search_padded_interval(self, chrom, pos, padding): interval_q = self._interval_query(chrom, start=max(pos - padding, MIN_POS), end=min(pos + padding, MAX_POS)) return self.filter(interval_q).result_values() diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 0400a9e809..9e11d0a9b8 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -439,7 +439,7 @@ def _search_dataset_type(search, genome_version): chroms = [gene[f'chromGrch{genome_version}'] for gene in (search.get('genes') or {}).values()] + [ interval['chrom'] for interval in (search.get('intervals') or []) - ] + ] if not search.get('exclude_locations') else None dataset_type = _annotation_dataset_type(search.get('annotations'), chroms, pathogenicity=search.get('pathogenicity')) secondary_dataset_type = _annotation_dataset_type(search['annotations_secondary'], chroms) if search.get('annotations_secondary') else None From 772ce1fc6ff9b3a5ada2ad2bbd8f88eda3be6481 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 16 Sep 2025 16:39:10 -0400 Subject: [PATCH 83/89] fix test --- seqr/utils/search/elasticsearch/es_utils_tests.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py index 787baed991..2c3a31302c 100644 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ b/seqr/utils/search/elasticsearch/es_utils_tests.py @@ -3018,13 +3018,14 @@ def test_skip_genotype_filter(self): expected_transcript_variant['selectedMainTranscriptId'] = PARSED_VARIANTS[1]['selectedMainTranscriptId'] self.assertListEqual(variants, [expected_transcript_variant, PARSED_MULTI_INDEX_VARIANT]) self.assertExecutedSearch( - index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME]), + index=','.join([INDEX_NAME, SECOND_INDEX_NAME]), filters=[{'terms': {'geneIds': ['ENSG00000228198']}}, ANNOTATION_QUERY], size=3, ) # test with inheritance override search_model.search['inheritance'] = {'mode': 'any_affected'} + search_model.search['locus'] = None search_model.save() _set_cache('search_results__{}__xpos'.format(results_model.guid), None) query_variants(results_model, num_results=2, skip_genotype_filter=True) @@ -3032,7 +3033,6 @@ def test_skip_genotype_filter(self): self.assertExecutedSearches([ dict( filters=[ - {'terms': {'geneIds': ['ENSG00000228198']}}, ANNOTATION_QUERY, {'bool': { 'should': [ @@ -3044,7 +3044,6 @@ def test_skip_genotype_filter(self): ], start_index=0, size=2, index=SECOND_INDEX_NAME), dict( filters=[ - {'terms': {'geneIds': ['ENSG00000228198']}}, ANNOTATION_QUERY, {'bool': { 'should': [ @@ -3056,7 +3055,6 @@ def test_skip_genotype_filter(self): ], start_index=0, size=2, index=MITO_WGS_INDEX_NAME), dict( filters=[ - {'terms': {'geneIds': ['ENSG00000228198']}}, ANNOTATION_QUERY, {'bool': { 'should': [ From 18fa557e920b74517c9366e393654307dfe520ee Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 16 Sep 2025 17:09:40 -0400 Subject: [PATCH 84/89] fix utils tests --- seqr/utils/search/search_utils_tests.py | 90 ++++++++----------------- 1 file changed, 29 insertions(+), 61 deletions(-) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 7b3fc58e89..1cfb2ebef1 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -316,14 +316,31 @@ def _test_expected_search_call(self, mock_get_variants, results_cache, search_fi rs_ids=None, variant_ids=None, parsed_variant_ids=None, inheritance_mode='de_novo', dataset_type=None, secondary_dataset_type=None, omitted_sample_guids=None, exclude_locations=False, exclude=None, annotations=None, annotations_secondary=None, single_gene_search=False, **kwargs): + genes = intervals = None + has_included_gene_search = has_gene_search and not exclude_locations + if has_gene_search: + genes = {'ENSG00000186092': mock.ANY, 'ENSG00000227232': mock.ANY} + intervals = [{'chrom': '2', 'start': 1234, 'end': 5678, 'offset': None}, {'chrom': '7', 'start': 100, 'end': 10100, 'offset': 0.1}] + if single_gene_search: + genes = {'ENSG00000227232': mock.ANY} + intervals = intervals[2:3] + expected_search = { 'inheritance_mode': inheritance_mode, 'inheritance_filter': {}, - 'parsed_locus': mock.ANY, 'skipped_samples': mock.ANY, - 'dataset_type': mock.ANY, + 'dataset_type': 'MITO_missing' if has_included_gene_search else dataset_type, 'secondary_dataset_type': secondary_dataset_type, + 'exclude_locations': exclude_locations, + 'genes': genes, + 'intervals': intervals, } + if not genes: + expected_search.update({ + 'parsed_variant_ids': parsed_variant_ids, + 'rs_ids': rs_ids, + 'variant_ids': variant_ids, + }) expected_search.update({field: self.search_model.search[field] for field in search_fields or []}) if exclude: expected_search['exclude'] = exclude @@ -335,19 +352,14 @@ def _test_expected_search_call(self, mock_get_variants, results_cache, search_fi mock_get_variants.assert_called_with(mock.ANY, expected_search, self.user, results_cache, '37', **kwargs) self._assert_expected_search_samples(mock_get_variants, omitted_sample_guids, has_gene_search and not exclude_locations) - gene_ids = intervals = None - has_included_gene_search = has_gene_search and not exclude_locations - if has_gene_search: - gene_ids = ['ENSG00000186092', 'ENSG00000227232'] - intervals = [['2', 1234, 5678], ['7', 1, 11100], ['1', 14404, 29570], ['1', 65419, 71585]] - if single_gene_search: - gene_ids = gene_ids[1:] - intervals = intervals[2:3] - self._assert_expected_search_locus( - mock_get_variants.call_args.args[1], dataset_type='MITO_missing' if has_included_gene_search else dataset_type, - gene_ids=gene_ids, intervals=intervals, rs_ids=rs_ids, variant_ids=variant_ids, - parsed_variant_ids=parsed_variant_ids, exclude_locations=exclude_locations, - ) + if genes: + parsed_genes = mock_get_variants.call_args.args[1]['genes'] + for gene in parsed_genes.values(): + self.assertSetEqual(set(gene.keys()), {'id', *GENE_FIELDS}) + self.assertEqual(parsed_genes['ENSG00000227232']['geneSymbol'], 'WASH7P') + if len(genes) > 1: + self.assertEqual(parsed_genes['ENSG00000186092']['geneSymbol'], 'OR4F5') + def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guids, has_gene_search): searched_samples = self.affected_search_samples @@ -361,12 +373,6 @@ def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guid self.assertSetEqual(set(mock_get_variants.call_args.args[1]['skipped_samples']), set(non_affected_search_samples)) - def _assert_expected_search_locus(self, search_body, dataset_type, **parsed_locus): - self.maxDiff = None - self.assertDictEqual(search_body['parsed_locus'], parsed_locus) - self.assertEqual(search_body['dataset_type'], dataset_type) - - def test_query_variants(self, mock_get_variants): def _mock_get_variants(families, search, user, previous_search_results, genome_version, **kwargs): previous_search_results['all_results'] = PARSED_VARIANTS @@ -430,14 +436,14 @@ def _mock_get_variants(families, search, user, previous_search_results, genome_v query_variants(self.results_model, user=self.user) self._test_expected_search_call( mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - has_gene_search=True, + has_gene_search=True, omitted_sample_guids=MITO_SAMPLES, ) self.search_model.search['locus']['rawItems'] = 'WASH7P' query_variants(self.results_model, user=self.user) self._test_expected_search_call( mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - has_gene_search=True, single_gene_search=True, + has_gene_search=True, single_gene_search=True, omitted_sample_guids=MITO_SAMPLES, ) del self.search_model.search['locus'] @@ -546,30 +552,6 @@ class ElasticsearchSearchUtilsTests(TestCase, SearchUtilsTests): def setUp(self): self.set_up() - def _assert_expected_search_locus(self, search_body, dataset_type, gene_ids=None, intervals=None, **kwargs): - genes = None - if gene_ids: - genes = { - gene_id: mock.ANY for gene_id in gene_ids - } - intervals = [ - {'chrom': '2', 'start': 1234, 'end': 5678, 'offset': None}, - {'chrom': '7', 'start': 100, 'end': 10100, 'offset': 0.1}, - ] if len(gene_ids) > 1 else [] - - dataset_type = None if dataset_type == 'MITO_missing' else dataset_type - super()._assert_expected_search_locus( - search_body, dataset_type, genes=genes, intervals=intervals, **kwargs, - ) - - if gene_ids: - parsed_genes = search_body['parsed_locus']['genes'] - for gene in parsed_genes.values(): - self.assertSetEqual(set(gene.keys()), {'id', *GENE_FIELDS}) - self.assertEqual(parsed_genes['ENSG00000227232']['geneSymbol'], 'WASH7P') - if len(gene_ids) > 1: - self.assertEqual(parsed_genes['ENSG00000186092']['geneSymbol'], 'OR4F5') - def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guids, has_gene_search): return super()._assert_expected_search_samples(mock_get_variants, omitted_sample_guids, False) @@ -666,20 +648,6 @@ def _assert_expected_cached_variants(self, variants, num_results): self.PARSED_CACHED_VARIANTS[:num_results], ) - def _assert_expected_search_locus(self, *args, gene_ids=None, intervals=None, exclude_locations=None, parsed_variant_ids=None, variant_ids=None, **kwargs): - gene_ids = None if exclude_locations else gene_ids - gene_intervals = None - if gene_ids: - if len(gene_ids) > 1: - gene_intervals = {2: intervals[2], 7: intervals[3]} - else: - gene_intervals = {2: intervals[0]} - intervals = intervals[:2] if len(gene_ids) > 1 else None - super()._assert_expected_search_locus( - *args, gene_ids=gene_ids, gene_intervals=gene_intervals, intervals=intervals, - variant_ids=parsed_variant_ids, exclude_intervals=exclude_locations, **kwargs, - ) - @mock.patch('seqr.utils.search.utils.get_clickhouse_variant_by_id') def test_get_single_variant(self, mock_call): mock_call.return_value = PARSED_VARIANTS[0] From e88b1e1d22c143e7112fbc8125a510405e809c94 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 16 Sep 2025 17:39:43 -0400 Subject: [PATCH 85/89] fix passed dataset type --- .../search/elasticsearch/es_utils_tests.py | 22 ++----------------- seqr/utils/search/search_utils_tests.py | 2 +- seqr/utils/search/utils.py | 2 +- 3 files changed, 4 insertions(+), 22 deletions(-) diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py index 2c3a31302c..2a899c34ef 100644 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ b/seqr/utils/search/elasticsearch/es_utils_tests.py @@ -2859,11 +2859,11 @@ def test_multi_project_prefilter_indices_get_es_variants(self): _, total_results = query_variants(results_model, num_results=2) self.assertEqual(total_results, 14) - self.assertTrue('index_metadata__{},{},{}'.format(INDEX_NAME, MITO_WGS_INDEX_NAME, SV_INDEX_NAME) in REDIS_CACHE) + self.assertTrue('index_metadata__{},{},{}'.format(INDEX_NAME, SECOND_INDEX_NAME, SV_INDEX_NAME) in REDIS_CACHE) gene_filter = {'terms': {'geneIds': ['ENSG00000228198']}} prefilter_search = dict( - filters=[gene_filter], index=f'{SV_INDEX_NAME},{MITO_WGS_INDEX_NAME},{SECOND_INDEX_NAME},{INDEX_NAME}', + filters=[gene_filter], index=f'{SV_INDEX_NAME},{SECOND_INDEX_NAME},{INDEX_NAME}', size=200, expected_source_fields=set(), ) sv_search = dict( @@ -2910,24 +2910,6 @@ def test_multi_project_prefilter_indices_get_es_variants(self): '_name': 'F000011_11' }} ], start_index=0, size=2, index=SECOND_INDEX_NAME), - dict(filters=[ - gene_filter, - { - 'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ]}}, {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - {'term': {'samples_gq_0_to_5': 'HG00733'}}, - {'term': {'samples_gq_5_to_10': 'HG00733'}}, - ]}} - ], - '_name': 'F000002_2' - }}], start_index=0, size=2, index=MITO_WGS_INDEX_NAME), dict(filters=[ gene_filter, {'bool': {'should': [ diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 1cfb2ebef1..128f23cfc7 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -329,7 +329,7 @@ def _test_expected_search_call(self, mock_get_variants, results_cache, search_fi 'inheritance_mode': inheritance_mode, 'inheritance_filter': {}, 'skipped_samples': mock.ANY, - 'dataset_type': 'MITO_missing' if has_included_gene_search else dataset_type, + 'dataset_type': None if has_included_gene_search else dataset_type, 'secondary_dataset_type': secondary_dataset_type, 'exclude_locations': exclude_locations, 'genes': genes, diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 9e11d0a9b8..3f46ce0a8d 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -309,7 +309,7 @@ def _query_variants(search_model, user, previous_search_results, genome_version, parsed_search[annotation_key] = {k: v for k, v in parsed_search[annotation_key].items() if v} dataset_type, secondary_dataset_type, lookup_dataset_type = _search_dataset_type(parsed_search, genome_version) - parsed_search.update({'dataset_type': dataset_type, 'secondary_dataset_type': secondary_dataset_type}) + parsed_search.update({'dataset_type': None if dataset_type == DATASET_TYPE_NO_MITO else dataset_type, 'secondary_dataset_type': secondary_dataset_type}) search_dataset_type = None if dataset_type and dataset_type != ALL_DATA_TYPES: if secondary_dataset_type is None or secondary_dataset_type == dataset_type: From 637c335977f0b465686ac16b3772637c5e7bb02b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 16 Sep 2025 18:49:56 -0400 Subject: [PATCH 86/89] clean up query caching --- seqr/utils/search/search_utils_tests.py | 27 ++--- seqr/utils/search/utils.py | 136 ++++++++++++++---------- 2 files changed, 95 insertions(+), 68 deletions(-) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 80bf5b7b08..3efb7df12d 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -51,6 +51,7 @@ def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): class SearchUtilsTests(SearchTestHelper): CACHED_VARIANTS = PARSED_VARIANTS + PARSED_VARIANTS + HAS_GENE_AGG = False def set_up(self): super(SearchUtilsTests, self).set_up() @@ -524,19 +525,13 @@ def _mock_get_gene_counts(families, search, user, previous_search_results, genom self.assertDictEqual(gene_counts, GENE_COUNTS) results_cache = {'all_results': self.GENE_AGG_ALL_RESULTS} if hasattr(self, 'GENE_AGG_ALL_RESULTS') else {'gene_aggs': gene_counts} self.assert_cached_results(results_cache) + kwargs = dict(sort=None, num_results=100) + if self.HAS_GENE_AGG: + kwargs['gene_agg'] = True self._test_expected_search_call( - mock_get_variants, results_cache, sort=None, num_results=100, gene_agg=True, + mock_get_variants, results_cache, **kwargs, ) - def test_cached_get_variant_query_gene_counts(self): - cached_gene_counts = { - 'ENSG00000135953': {'total': 5, 'families': {'F000003_3': 2, 'F000002_2': 1, 'F000011_11': 4}}, - 'ENSG00000228198': {'total': 5, 'families': {'F000003_3': 4, 'F000002_2': 1, 'F000011_11': 4}} - } - self.set_cache({'total_results': 5, 'gene_aggs': cached_gene_counts}) - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, cached_gene_counts) - @mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') @mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') @@ -544,6 +539,8 @@ class ElasticsearchSearchUtilsTests(TestCase, SearchUtilsTests): databases = ['default', 'reference_data'] fixtures = ['users', '1kg_project', 'reference_data'] + HAS_GENE_AGG = True + def setUp(self): self.set_up() @@ -621,7 +618,13 @@ def test_get_variant_query_gene_counts(self, mock_get_variants): super(ElasticsearchSearchUtilsTests, self).test_get_variant_query_gene_counts(mock_get_variants) def test_cached_get_variant_query_gene_counts(self): - super(ElasticsearchSearchUtilsTests, self).test_cached_get_variant_query_gene_counts() + cached_gene_counts = { + 'ENSG00000135953': {'total': 5, 'families': {'F000003_3': 2, 'F000002_2': 1, 'F000011_11': 4}}, + 'ENSG00000228198': {'total': 5, 'families': {'F000003_3': 4, 'F000002_2': 1, 'F000011_11': 4}} + } + self.set_cache({'total_results': 5, 'gene_aggs': cached_gene_counts}) + gene_counts = get_variant_query_gene_counts(self.results_model, self.user) + self.assertDictEqual(gene_counts, cached_gene_counts) self.set_cache({'all_results': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT, 'total_results': 2}) gene_counts = get_variant_query_gene_counts(self.results_model, self.user) @@ -733,8 +736,6 @@ def test_get_variant_query_gene_counts(self, mock_call): super().test_get_variant_query_gene_counts(mock_call) def test_cached_get_variant_query_gene_counts(self): - super().test_cached_get_variant_query_gene_counts() - self.set_cache({'all_results': self.CACHED_VARIANTS + [SV_VARIANT1], 'total_results': 5}) gene_counts = get_variant_query_gene_counts(self.results_model, self.user) self.assertDictEqual(gene_counts, { diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index ca50f8fed9..7ccc14e573 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -241,23 +241,14 @@ def _get_search_cache_key(search_model, sort=None): return 'search_results__{}__{}'.format(search_model.guid, sort or XPOS_SORT_KEY) -def _process_clickhouse_unsorted_cached_results(cache_key, sort, family_guid): - unsorted_results = safe_redis_get_wildcard_json(cache_key.replace(sort or 'xpos', '*')) - if not unsorted_results: - return None - results = get_clickhouse_cache_results(unsorted_results['all_results'], sort, family_guid) - safe_redis_set_json(cache_key, results, expire=timedelta(weeks=2)) - return results +def _get_any_sort_cached_results(search_model): + cache_key = _get_search_cache_key(search_model, sort='*') + return safe_redis_get_wildcard_json(cache_key) -def _get_cached_search_results(search_model, sort=None, family_guid=None): +def _get_cached_search_results(search_model, sort=None): cache_key = _get_search_cache_key(search_model, sort=sort) - results = safe_redis_get_json(cache_key) - if not results: - results = backend_specific_call( # TODO - lambda *args: None, _process_clickhouse_unsorted_cached_results, - )(cache_key, sort, family_guid) - return results or {} + return safe_redis_get_json(cache_key) or {} def _validate_export_variant_count(total_variants): @@ -265,10 +256,42 @@ def _validate_export_variant_count(total_variants): raise InvalidSearchException(f'Unable to export more than {MAX_EXPORT_VARIANTS} variants ({total_variants} requested)') -def query_variants(search_model, sort=XPOS_SORT_KEY, skip_genotype_filter=False, load_all=False, user=None, page=1, num_results=100): - previous_search_results = _get_cached_search_results(search_model, sort=sort, family_guid=search_model.families.first().guid) - total_results = previous_search_results.get('total_results') +def _get_elasticsearch_previous_search_results(search_model, sort, page, num_results, load_all, **kwargs): + previous_search_results = _get_cached_search_results(search_model, sort=sort) + start_index, end_index, num_results = _get_result_range(page, num_results, previous_search_results.get('total_results'), load_all) + + cached_page = None + loaded_results = previous_search_results.get('all_results') or [] + if len(loaded_results) >= end_index: + cached_page = loaded_results[start_index:end_index] + + if not cached_page: + cached_page = process_es_previously_loaded_results(previous_search_results, start_index, end_index) + + return previous_search_results, cached_page, num_results + +def _get_clickhouse_previous_search_results(search_model, sort, page, num_results, load_all, genome_version=None): + previous_search_results = _get_cached_search_results(search_model, sort=sort) + if not previous_search_results: + unsorted_results = _get_any_sort_cached_results(search_model) + if unsorted_results: + previous_search_results = get_clickhouse_cache_results( + unsorted_results['all_results'], sort, family_guid=search_model.families.first().guid, + ) + cache_key = _get_search_cache_key(search_model, sort=sort) + safe_redis_set_json(cache_key, previous_search_results, expire=timedelta(weeks=2)) + + start_index, end_index, num_results = _get_result_range(page, num_results, previous_search_results.get('total_results'), load_all) + cached_page = None + loaded_results = previous_search_results.get('all_results') or [] + if len(loaded_results) >= end_index: + cached_page = format_clickhouse_results(loaded_results[start_index:end_index], genome_version) + + return previous_search_results, cached_page, num_results + + +def _get_result_range(page, num_results, total_results, load_all): if load_all: num_results = total_results or MAX_EXPORT_VARIANTS _validate_export_variant_count(num_results) @@ -277,24 +300,21 @@ def query_variants(search_model, sort=XPOS_SORT_KEY, skip_genotype_filter=False, if total_results is not None: end_index = min(end_index, total_results) - genome_version = _get_search_genome_version(search_model.families.all()) - loaded_results = previous_search_results.get('all_results') or [] - if len(loaded_results) >= end_index: - results_page = backend_specific_call( # TODO - lambda results, genome_version: results, format_clickhouse_results, - )(loaded_results[start_index:end_index], genome_version) - return results_page, total_results - - previously_loaded_results = backend_specific_call( # TODO - process_es_previously_loaded_results, - lambda *args: None, # Other backends need no additional parsing - )(previous_search_results, start_index, end_index) - if previously_loaded_results is not None: - return previously_loaded_results, total_results - if end_index > MAX_VARIANTS: raise InvalidSearchException(f'Unable to load more than {MAX_VARIANTS} variants ({end_index} requested)') + return start_index, end_index, num_results + + +def query_variants(search_model, sort=XPOS_SORT_KEY, skip_genotype_filter=False, load_all=False, user=None, page=1, num_results=100): + genome_version = _get_search_genome_version(search_model.families.all()) + previous_search_results, cached_page, num_results = backend_specific_call( + _get_elasticsearch_previous_search_results, + _get_clickhouse_previous_search_results, + )(search_model, sort, page, num_results, load_all, genome_version=genome_version) + if cached_page is not None: + return cached_page, previous_search_results.get('total_results') + variants, total_results = _query_variants( search_model, user, previous_search_results, genome_version, sort=sort, page=page, num_results=num_results, skip_genotype_filter=skip_genotype_filter) @@ -383,44 +403,50 @@ def _execute_search(samples, parsed_search, user, previous_search_results, genom def get_variant_query_gene_counts(search_model, user): + return backend_specific_call( + _get_es_variant_query_gene_counts, + _get_clickhouse_variant_query_gene_counts, + )(search_model, user) + + +def _get_es_variant_query_gene_counts(search_model, user): previous_search_results = _get_cached_search_results(search_model) if previous_search_results.get('gene_aggs'): return previous_search_results['gene_aggs'] if len(previous_search_results.get('all_results', [])) == previous_search_results.get('total_results'): - return _get_gene_aggs_for_cached_variants(previous_search_results) + return _get_gene_aggs_for_cached_variants( + previous_search_results['all_results'], + lambda v: next(( + [gene_id] for gene_id, transcripts in v['transcripts'].items() + if any(t['transcriptId'] == v['mainTranscriptId'] for t in transcripts) + ), []) if v['mainTranscriptId'] else [], + ) - previously_loaded_results = backend_specific_call( # TODO - process_es_previously_loaded_gene_aggs, - lambda *args: None, # Other backends need no additional parsing - )(previous_search_results) + previously_loaded_results = process_es_previously_loaded_gene_aggs(previous_search_results) if previously_loaded_results is not None: return previously_loaded_results genome_version = _get_search_genome_version(search_model.families.all()) gene_counts, _ = _query_variants(search_model, user, previous_search_results, genome_version, gene_agg=True) - gene_counts = backend_specific_call( # TODO - lambda *args: None, _get_gene_aggs_for_cached_variants, - )(previous_search_results) or gene_counts return gene_counts -def _get_gene_aggs_for_cached_variants(previous_search_results): +def _get_clickhouse_variant_query_gene_counts(search_model, user): + previous_search_results = _get_any_sort_cached_results(search_model) or {} + if len(previous_search_results.get('all_results', [])) != previous_search_results.get('total_results'): + genome_version = _get_search_genome_version(search_model.families.all()) + _query_variants(search_model, user, previous_search_results, genome_version) + + return _get_gene_aggs_for_cached_variants([ + v for variants in previous_search_results['all_results'] for v in (variants if isinstance(variants, list) else [variants]) + ], lambda v: v['transcripts'].keys() if 'transcripts' in v else {t['geneId'] for t in v['sortedTranscriptConsequences']}) + + +def _get_gene_aggs_for_cached_variants(variants, get_variant_genes): gene_aggs = defaultdict(lambda: {'total': 0, 'families': defaultdict(int)}) - # ES caches compound hets separately from main results, other backends cache everything together - flattened_variants = backend_specific_call( # TODO - lambda results: results, - lambda results: [v for variants in results for v in (variants if isinstance(variants, list) else [variants])], - )(previous_search_results['all_results']) - for var in flattened_variants: - # ES only reports breakdown for main transcript gene only, other backends report for all genes - gene_ids = backend_specific_call( # TODO - lambda variant: next(( - [gene_id] for gene_id, transcripts in variant['transcripts'].items() - if any(t['transcriptId'] == var['mainTranscriptId'] for t in transcripts) - ), []) if var['mainTranscriptId'] else [], - lambda variant: variant['transcripts'].keys() if 'transcripts' in variant else {t['geneId'] for t in variant['sortedTranscriptConsequences']}, - )(var) + for var in variants: + gene_ids = get_variant_genes(var) for gene_id in gene_ids: gene_aggs[gene_id]['total'] += 1 for family_guid in var['familyGuids']: From 1bf016aab83f291b5f5b5fe95f4553d653e07d5a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 16 Sep 2025 18:54:58 -0400 Subject: [PATCH 87/89] clean up unneccessary helper func --- seqr/utils/search/utils.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 85b7c3c6e4..cdd28df570 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -329,7 +329,7 @@ def _query_variants(search_model, user, previous_search_results, genome_version, _validate_search(parsed_search, samples, previous_search_results) - variant_results = _execute_search( + variant_results = backend_specific_call(get_es_variants, get_clickhouse_variants)( samples, parsed_search, user, previous_search_results, genome_version, sort=sort, num_results=num_results, **kwargs, ) @@ -340,12 +340,6 @@ def _query_variants(search_model, user, previous_search_results, genome_version, return variant_results, previous_search_results.get('total_results') -def _execute_search(samples, parsed_search, user, previous_search_results, genome_version, **kwargs): - return backend_specific_call(get_es_variants, get_clickhouse_variants)( - samples, parsed_search, user, previous_search_results, genome_version, **kwargs, - ) - - def get_variant_query_gene_counts(search_model, user): previous_search_results = _get_cached_search_results(search_model) if previous_search_results.get('gene_aggs'): From 1bd71e39108533616ea4dd7b54871b7b653dc962 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 16 Sep 2025 19:44:51 -0400 Subject: [PATCH 88/89] fix test --- seqr/utils/search/elasticsearch/es_utils_tests.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py index 787baed991..968e64724c 100644 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ b/seqr/utils/search/elasticsearch/es_utils_tests.py @@ -1513,6 +1513,7 @@ def test_invalid_get_es_variants(self, mock_logger): query_variants(results_model) self.assertEqual(str(cm.exception), 'Could not find expected indices: test_index_sv, test_index_mito_wgs, test_index') + @mock.patch('seqr.utils.search.utils.MAX_VARIANTS', 100) @urllib3_responses.activate def test_get_es_variants(self): setup_responses() @@ -1554,16 +1555,14 @@ def test_get_es_variants(self): # test load_all setup_responses() - with mock.patch('seqr.utils.search.utils.MAX_VARIANTS', 100): - variants, _ = query_variants(results_model, page=1, num_results=2, load_all=True) + variants, _ = query_variants(results_model, page=1, num_results=2, load_all=True) self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY], start_index=4, size=1) self.assertEqual(len(variants), 5) self.assertListEqual(variants, PARSED_VARIANTS + PARSED_VARIANTS + PARSED_VARIANTS[:1]) # test does not re-fetch once all loaded urllib3_responses.reset() - with mock.patch('seqr.utils.search.utils.MAX_VARIANTS', 1): - variants, _ = query_variants(results_model, page=1, num_results=2, load_all=True) + variants, _ = query_variants(results_model, page=1, num_results=2, load_all=True) self.assertEqual(len(variants), 5) self.assertListEqual(variants, PARSED_VARIANTS + PARSED_VARIANTS + PARSED_VARIANTS[:1]) From 465aa46498d5ce8560f194c99c2dea1d65949d89 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 18 Sep 2025 13:15:04 -0400 Subject: [PATCH 89/89] bump changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60dc4166da..3045cb08ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## dev +## 9/18/25 +# Deprecate hail backend support + ## 8/7/25 * Deprecate support for lifting existing projects from GRCH37 to GRCh38