diff --git a/.cloudbuild/seqr-hail-search-docker.cloudbuild.yaml b/.cloudbuild/seqr-hail-search-docker.cloudbuild.yaml deleted file mode 100644 index 91778b7b1c..0000000000 --- a/.cloudbuild/seqr-hail-search-docker.cloudbuild.yaml +++ /dev/null @@ -1,11 +0,0 @@ -steps: -- name: 'gcr.io/kaniko-project/executor:v1.3.0' - args: - - --destination=gcr.io/seqr-project/seqr-hail-search:${COMMIT_SHA} - - --destination=gcr.io/seqr-project/seqr-hail-search:${_CUSTOM_BRANCH_TAG} - - --destination=gcr.io/seqr-project/seqr-hail-search:latest - - --dockerfile=hail_search/deploy/Dockerfile - - --cache=true - - --cache-ttl=168h - -timeout: 1800s diff --git a/.dockerignore b/.dockerignore index e1a4f868bb..e294067478 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,9 +5,7 @@ static/* !static/images/* deploy/* !deploy/docker/seqr/* -!deploy/docker/hail_search/* !deploy/docker/vlm/* -hail_search/fixtures/* .git .vscode .idea diff --git a/.github/workflows/dev-hail-search-release.yaml b/.github/workflows/dev-hail-search-release.yaml deleted file mode 100644 index afd197cf39..0000000000 --- a/.github/workflows/dev-hail-search-release.yaml +++ /dev/null @@ -1,64 +0,0 @@ -name: seqr hail search dev release -on: - workflow_run: - workflows: ["Hail Search Unit Tests"] - types: - - completed - branches: - - dev - -permissions: - id-token: write - -jobs: - docker: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - name: checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.event.workflow_run.head_branch }} - - - name: authenticate to google cloud - id: "auth" - uses: google-github-actions/auth@v0 - with: - workload_identity_provider: "${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}" - service_account: "${{ secrets.RUN_SA_EMAIL }}" - - - name: "setup gcloud sdk" - uses: google-github-actions/setup-gcloud@v0 - - - name: Build and push images - run: |- - gcloud builds submit --quiet --substitutions="COMMIT_SHA=${{ github.event.workflow_run.head_sha }},_CUSTOM_BRANCH_TAG=gcloud-dev" --config .cloudbuild/seqr-hail-search-docker.cloudbuild.yaml --gcs-log-dir=gs://seqr-github-actions-logs/logs . - - helm_update: - runs-on: ubuntu-latest - needs: docker - steps: - - name: Retrieve tgg-helm repo for broad seqr chart - uses: actions/checkout@v3 - with: - repository: broadinstitute/tgg-helm - token: ${{ secrets.SEQR_VERSION_UPDATE_TOKEN }} - ref: main - persist-credentials: false - fetch-depth: 0 - - - name: update image tag in the dev broad seqr chart - uses: mikefarah/yq@v4.22.1 - with: - cmd: > - yq -i '.seqr-platform.hail-search.image.tag = "${{ github.event.workflow_run.head_sha }}"' charts/dev-broad-seqr/values.yaml - - - name: Commit and Push changes - uses: Andro999b/push@v1.3 - with: - repository: broadinstitute/tgg-helm - branch: main - github_token: ${{ secrets.SEQR_VERSION_UPDATE_TOKEN }} - author_email: ${{ github.actor }}@users.noreply.github.com - author_name: tgg-automation - message: "Update hail-search dev release docker tag to ${{ github.event.workflow_run.head_sha }}" diff --git a/.github/workflows/docker-lint.yaml b/.github/workflows/docker-lint.yaml index b928c7c161..67a481e784 100644 --- a/.github/workflows/docker-lint.yaml +++ b/.github/workflows/docker-lint.yaml @@ -9,7 +9,6 @@ on: - master paths: - deploy/docker/seqr/Dockerfile - - hail_search/deploy/Dockerfile - vlm/deploy/Dockerfile - .hadolint.yaml - .docker-compose.yaml @@ -21,7 +20,6 @@ on: - master paths: - deploy/docker/seqr/Dockerfile - - hail_search/deploy/Dockerfile - vlm/deploy/Dockerfile - .hadolint.yaml - .docker-compose.yaml @@ -37,9 +35,6 @@ jobs: - uses: hadolint/hadolint-action@v1.5.0 with: dockerfile: deploy/docker/seqr/Dockerfile - - uses: hadolint/hadolint-action@v1.5.0 - with: - dockerfile: hail_search/deploy/Dockerfile - uses: hadolint/hadolint-action@v1.5.0 with: dockerfile: vlm/deploy/Dockerfile diff --git a/.github/workflows/hail-search-unit-tests.yaml b/.github/workflows/hail-search-unit-tests.yaml deleted file mode 100644 index 389afdc3e7..0000000000 --- a/.github/workflows/hail-search-unit-tests.yaml +++ /dev/null @@ -1,38 +0,0 @@ -name: Hail Search Unit Tests - -# Run the test suite on pushes (incl. merges) to master and dev -# Run the test suite when a PR is opened, pushed to, or reopened -on: - push: - branches: - - master - - dev - paths: - - 'hail_search/**' - pull_request: - types: [opened, synchronize, reopened] - paths: - - 'hail_search/**' - -jobs: - hail_search: - runs-on: ubuntu-latest - container: hailgenetics/hail:0.2.128 - - steps: - - uses: actions/checkout@v2 - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip wheel - pip install -r hail_search/requirements-test.txt - - name: Run coverage tests - run: | - export HAIL_SEARCH_DATA_DIR=./hail_search/fixtures - export REFERENCE_DATASETS_DIR=./hail_search/fixtures/reference_data - export MAX_GENE_INTERVALS=3 - export MACHINE_MEM=24 - export JAVA_OPTS_XSS=16M - coverage run --source="./hail_search" --omit="./hail_search/__main__.py","./hail_search/test_utils.py" -m pytest hail_search/ - coverage report --fail-under=99 - coverage report | (! grep [1-8][0-9]%) - coverage report | (! grep 9[0-5]%) diff --git a/.github/workflows/prod-hail-search-release.yaml b/.github/workflows/prod-hail-search-release.yaml deleted file mode 100644 index 0a2dbb0ee7..0000000000 --- a/.github/workflows/prod-hail-search-release.yaml +++ /dev/null @@ -1,79 +0,0 @@ -name: seqr hail search prod release -on: - workflow_run: - workflows: ["Hail Search Unit Tests"] - types: - - completed - branches: - - master - -permissions: - id-token: write - -jobs: - docker: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - name: checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.event.workflow_run.head_branch }} - - - name: authenticate to google cloud - id: "auth" - uses: google-github-actions/auth@v0 - with: - workload_identity_provider: "${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}" - service_account: "${{ secrets.RUN_SA_EMAIL }}" - - - name: "setup gcloud sdk" - uses: google-github-actions/setup-gcloud@v0 - - - name: Build and push images - run: |- - gcloud builds submit --quiet --substitutions="COMMIT_SHA=${{ github.event.workflow_run.head_sha }},_CUSTOM_BRANCH_TAG=gcloud-prod" --config .cloudbuild/seqr-hail-search-docker.cloudbuild.yaml --gcs-log-dir=gs://seqr-github-actions-logs/logs . - - helm_update: - runs-on: ubuntu-latest - needs: docker - steps: - - uses: actions/checkout@v3 - with: - repository: broadinstitute/seqr-helm - ref: main - persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token - fetch-depth: 0 # otherwise, you will failed to push refs to dest repo - - - name: Get latest hail-search version - uses: mikefarah/yq@v4.22.1 - id: current - with: - cmd: > - yq -r '.version' charts/hail-search/Chart.yaml - - - name: Bump version - id: bump - uses: cbrgm/semver-bump-action@main - with: - current-version: ${{ steps.current.outputs.result }} - bump-level: minor - - - name: Update appVersion and version in seqr Chart file - uses: mikefarah/yq@v4.22.1 - with: - cmd: > - yq -i ' - .appVersion = "${{ github.event.workflow_run.head_sha }}" | - .version = "${{ steps.bump.outputs.new_version }}" - ' charts/hail-search/Chart.yaml - - - name: Commit and Push changes - uses: Andro999b/push@v1.3 - with: - repository: broadinstitute/seqr-helm - branch: main - github_token: ${{ secrets.SEQR_VERSION_UPDATE_TOKEN }} - author_email: ${{ github.actor }}@users.noreply.github.com - author_name: tgg-automation - message: "Update hail-search chart appVersion to ${{ github.event.workflow_run.head_sha }}" diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6a8e929bfa..56368fb8ef 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -8,16 +8,12 @@ on: - master - dev paths-ignore: - - 'hail_search/**' - 'vlm/**' - - '.github/workflows/*hail-search*.yaml' - '.github/workflows/*vlm*.yaml' pull_request: types: [opened, synchronize, reopened] paths-ignore: - - 'hail_search/**' - 'vlm/**' - - '.github/workflows/*hail-search*.yaml' - '.github/workflows/*vlm*.yaml' jobs: @@ -73,11 +69,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - uses: actions/setup-java@v1 - with: - java-version: "8" # The JDK version needed by hail - java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk - architecture: x64 # (x64 or x86) - defaults to x64 - name: Use pip cache uses: actions/cache@v4 with: @@ -118,5 +109,7 @@ jobs: - run: npm install -g npm@7.24.2 - run: npm ci working-directory: ./ui + - run: npm run lint + working-directory: ./ui - run: npm test working-directory: ./ui diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 35c1c631e8..c5a570784e 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -12,35 +12,13 @@ on: paths: - 'vlm/**' - '.github/workflows/*vlm*.yaml' - - 'hail_search/fixtures/*' pull_request: types: [opened, synchronize, reopened] paths: - 'vlm/**' - '.github/workflows/*vlm*.yaml' - - 'hail_search/fixtures/*' jobs: - vlm_hail_backend: - runs-on: ubuntu-latest - container: hailgenetics/hail:0.2.128 - - steps: - - uses: actions/checkout@v2 - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip wheel - pip install -r vlm/requirements-test.txt - - name: Run coverage tests - run: | - export VLM_DATA_DIR=./hail_search/fixtures - export SEQR_BASE_URL=https://test-seqr.org/ - export NODE_ID=TestVLM - export MACHINE_MEM=24 - export JAVA_OPTS_XSS=16M - coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py","./vlm/clickhouse_utils.py" -m pytest vlm/ - coverage report --fail-under=95 - vlm_clickhouse: runs-on: ubuntu-latest container: hailgenetics/hail:0.2.128 @@ -78,6 +56,6 @@ jobs: export CLICKHOUSE_VLM_USERNAME=vlm_test_user export CLICKHOUSE_VLM_PASSWORD=vlm_test_password export CLICKHOUSE_DATABASE=test_seqr - coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py","./vlm/hail_backend_utils.py" -m pytest vlm/ + coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py" -m pytest vlm/ coverage report --fail-under=95 diff --git a/CHANGELOG.md b/CHANGELOG.md index 60dc4166da..3045cb08ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## dev +## 9/18/25 +# Deprecate hail backend support + ## 8/7/25 * Deprecate support for lifting existing projects from GRCH37 to GRCh38 diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 1f8528ed7c..7351b83809 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -54,6 +54,11 @@ def _seqr_pop_fields(self, seqr_populations): seqr_pop_fields += [f"'{sub_fields['hom']}_{sample_type}'" for sample_type in sample_types] return seqr_pop_fields + def _format_gene_intervals(self, genes): + return [ + {field: gene[f'{field}Grch{self.genome_version}'] for field in ['chrom', 'start', 'end']} for gene in genes.values() + ] + class AnnotationsQuerySet(SearchQuerySet): @@ -154,6 +159,10 @@ def clinvar_model(self): def clinvar_field_prefix(self): return f'{self.entry_field}__clinvar_join' + @property + def genome_version(self): + return self.model.ANNOTATION_CONSTANTS['genomeVersion'] + def subquery_join(self, subquery, join_key='key'): join_field = next(field for field in subquery.model._meta.fields if field.name == join_key) @@ -205,12 +214,11 @@ def _get_join_query_values(self, query, alias, conditional_selects): **{f'{alias}_{field}': value for field, value in query_select.items()}, ) - def search(self, parsed_locus=None, **kwargs): - parsed_locus = parsed_locus or {} - results = self.filter_variant_ids(**parsed_locus) + def search(self, **kwargs): + results = self.filter_variant_ids(**kwargs) results = self._filter_frequency(results, **kwargs) results = self._filter_in_silico(results, **kwargs) - results = self.filter_annotations(results, **parsed_locus, **kwargs) + results = self.filter_annotations(results, **kwargs) return results def result_values(self, skip_entry_fields=False): @@ -323,11 +331,11 @@ def search_compound_hets(self, primary_q, secondary_q): **{primary_gene_field: F(secondary_gene_field)} ).exclude(primary_variantId=F('secondary_variantId')) - def filter_variant_ids(self, variant_ids=None, rs_ids=None, **kwargs): + def filter_variant_ids(self, parsed_variant_ids=None, rs_ids=None, **kwargs): results = self - if variant_ids: + if parsed_variant_ids: results = results.filter( - variant_id__in=[f'{chrom}-{pos}-{ref}-{alt}' for chrom, pos, ref, alt in variant_ids] + variant_id__in=[f'{chrom}-{pos}-{ref}-{alt}' for chrom, pos, ref, alt in parsed_variant_ids] ) if rs_ids: @@ -413,7 +421,7 @@ def _get_in_silico_score_q(self, score, value): except ValueError: return Q(**{score_column: value}) - def filter_annotations(self, results, annotations=None, pathogenicity=None, exclude=None, gene_ids=None, intervals=None, exclude_intervals=False, padded_interval=None, **kwargs): + def filter_annotations(self, results, annotations=None, pathogenicity=None, exclude=None, genes=None, intervals=None, exclude_locations=False, padded_interval_end=None, **kwargs): transcript_field = self.transcript_field if self.model.GENOTYPE_OVERRIDE_FIELDS: results = results.annotate(**{ @@ -422,25 +430,28 @@ def filter_annotations(self, results, annotations=None, pathogenicity=None, excl }]), }) transcript_field = self.GENOTYPE_GENE_CONSEQUENCE_FIELD - interval_qs = [self._interval_query(*interval) for interval in intervals or []] - if gene_ids: + interval_qs = [self._interval_query(**interval) for interval in intervals or []] + if exclude_locations and genes: + interval_qs += [self._interval_query(**interval) for interval in self._format_gene_intervals(genes)] + genes = None + if genes: results = results.annotate(**{ self.GENE_CONSEQUENCE_FIELD: ArrayFilter(transcript_field, conditions=[{ - 'geneId': (gene_ids, 'has({value}, {field})'), + 'geneId': (sorted(genes.keys()), 'has({value}, {field})'), }]), }) gene_q = Q(gene_consequences__not_empty=True) for interval_q in interval_qs: gene_q |= interval_q results = results.filter(gene_q) - elif (interval_qs or padded_interval) and hasattr(self.model, 'end'): - if padded_interval: - padding = int((padded_interval['end'] - padded_interval['start']) * padded_interval['padding']) - interval_qs = [Q(end__range=(padded_interval['end'] - padding, padded_interval['end'] + padding))] + elif (interval_qs or padded_interval_end) and hasattr(self.model, 'end'): + if padded_interval_end: + end, padding = padded_interval_end + interval_qs = [Q(end__range=(end - padding, end + padding))] interval_q = interval_qs[0] for q in interval_qs[1:]: interval_q |= q - filter_func = results.exclude if exclude_intervals else results.filter + filter_func = results.exclude if exclude_locations else results.filter results = filter_func(interval_q) filter_qs, transcript_filters = self._parse_annotation_filters(annotations, pathogenicity) if (annotations or pathogenicity) else ([], []) @@ -463,7 +474,7 @@ def filter_annotations(self, results, annotations=None, pathogenicity=None, excl filter_q = Q(passes_annotation=True) if transcript_filters: - consequence_field = self.GENE_CONSEQUENCE_FIELD if gene_ids else transcript_field + consequence_field = self.GENE_CONSEQUENCE_FIELD if genes else transcript_field results = results.annotate(**{ self.FILTERED_CONSEQUENCE_FIELD: ArrayFilter(consequence_field, conditions=transcript_filters), }) @@ -475,7 +486,11 @@ def filter_annotations(self, results, annotations=None, pathogenicity=None, excl return results.filter(filter_q) - def _interval_query(self, chrom, start, end): + def _interval_query(self, chrom, start, end, offset=None, **kwargs): + if offset: + offset_pos = int((end - start) * offset) + start = max(start - offset_pos, MIN_POS) + end = min(end + offset_pos, MAX_POS) q = Q(xpos__range=(get_xpos(chrom, start), get_xpos(chrom, end))) if hasattr(self.model, 'end_chrom'): q |= Q(end_chrom__isnull=True, chrom=chrom, end__range=(start, end)) @@ -683,8 +698,12 @@ def genotype_fields(self): def clinvar_model(self): return self.model.clinvar_join.rel.related_model - def search(self, sample_data, parsed_locus=None, freqs=None, annotations=None, **kwargs): - entries = self.filter_locus(**(parsed_locus or {})) + @property + def genome_version(self): + return self.annotations_model.ANNOTATION_CONSTANTS['genomeVersion'] + + def search(self, sample_data, freqs=None, annotations=None, **kwargs): + entries = self.filter_locus(**kwargs) entries = self._join_annotations(entries) @@ -724,7 +743,7 @@ def _join_annotations(self, entries): return entries - def result_values(self, sample_data): + def result_values(self, sample_data=None): entries = self._join_annotations(self) if sample_data: return self._search_call_data(entries, sample_data) @@ -1043,49 +1062,46 @@ def _family_passes_expression(pass_field): mapped_expression='x.1', output_field=models.ArrayField(models.StringField()), ) - def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=None, gene_ids=None, variant_ids=None, padded_interval=None, require_gene_filter=False, **kwargs): + def filter_locus(self, exclude_locations=False, intervals=None, genes=None, parsed_variant_ids=None, require_gene_filter=False, **kwargs): entries = self - if variant_ids: + if parsed_variant_ids: # although technically redundant, the interval query is applied to the entries table before join and reduces the join size, # while the full variant_id filter is applied to the annotation table after the join - intervals = [(chrom, pos, pos) for chrom, pos, _, _ in variant_ids] + intervals = [{'chrom': chrom, 'start': pos, 'end': pos} for chrom, pos, _, _ in parsed_variant_ids] - if padded_interval: - pos = padded_interval['start'] - padding = int((padded_interval['end'] - pos) * padded_interval['padding']) - intervals = [(padded_interval['chrom'], max(pos - padding, MIN_POS), min(pos + padding, MAX_POS))] - elif 'cn' in self.call_fields: + should_filter_interval = False + if 'cn' in self.call_fields: # SV interval filtering occurs after joining on annotations to correctly incorporate end position - if exclude_intervals: + if exclude_locations: intervals = None else: - if gene_ids and 'geneIds' in self.call_fields: + chromosomes = {i['chrom'] for i in (intervals or [])} + if genes and 'geneIds' in self.call_fields: entries = entries.filter(calls__array_all={'OR': [ - {'geneIds': (gene_ids, 'hasAny({value}, {field})')}, + {'geneIds': (list(genes.keys()), 'hasAny({value}, {field})')}, {'gt': (None, 'isNull({field})')}, ]}) - intervals = list((gene_intervals or {}).values()) + (intervals or []) - gene_intervals = None - chromosomes = {chrom for chrom, _, _ in (intervals or [])} - intervals = [(chrom, MIN_POS, MAX_POS) for chrom in chromosomes] + chromosomes.update({gene[f'chromGrch{self.genome_version}'] for gene in genes.values()}) + should_filter_interval = True + intervals = [{'chrom': chrom, 'start': MIN_POS, 'end': MAX_POS} for chrom in chromosomes] - if not (gene_intervals or intervals): + if not (genes or intervals): return entries locus_q = None - if gene_intervals: + if genes: if hasattr(self.model, 'is_annotated_in_any_gene') and not intervals: entries = entries.filter(is_annotated_in_any_gene=Value(True)) - should_filter_interval = (not hasattr(self.model, 'geneId_ids')) or exclude_intervals or len(gene_intervals) < self.model.MAX_XPOS_FILTER_INTERVALS + should_filter_interval |= (not hasattr(self.model, 'geneId_ids')) or exclude_locations or len(genes) < self.model.MAX_XPOS_FILTER_INTERVALS if should_filter_interval: - intervals = list((gene_intervals or {}).values()) + (intervals or []) + intervals = self._format_gene_intervals(genes) + (intervals or []) if require_gene_filter or (not should_filter_interval): - locus_q = Q(geneId_ids__bitmap_has_any=list(gene_intervals.keys())) + locus_q = Q(geneId_ids__bitmap_has_any=[gene['id'] for gene in genes.values()]) if intervals: - interval_q = self._interval_query(*intervals[0]) + interval_q = self._interval_query(**intervals[0]) for interval in intervals[1:]: - interval_q |= self._interval_query(*interval) + interval_q |= self._interval_query(**interval) if locus_q is None: locus_q = interval_q elif require_gene_filter: @@ -1093,11 +1109,19 @@ def filter_locus(self, exclude_intervals=False, intervals=None, gene_intervals=N else: locus_q |= interval_q - filter_func = entries.exclude if exclude_intervals else entries.filter + filter_func = entries.exclude if exclude_locations else entries.filter return filter_func(locus_q) + def search_padded_interval(self, chrom, pos, padding): + interval_q = self._interval_query(chrom, start=max(pos - padding, MIN_POS), end=min(pos + padding, MAX_POS)) + return self.filter(interval_q).result_values() + @staticmethod - def _interval_query(chrom, start, end): + def _interval_query(chrom, start, end, offset=None, **kwargs): + if offset: + offset_pos = int((end - start) * offset) + start = max(start - offset_pos, MIN_POS) + end = min(end + offset_pos, MAX_POS) return Q(xpos__range=(get_xpos(chrom, start), get_xpos(chrom, end))) def _filter_seqr_frequency(self, entries, ac=None, hh=None, **kwargs): diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 1d09cd23cc..fd8106fea8 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -29,10 +29,6 @@ SELECTED_TRANSCRIPT_FIELD = 'selectedTranscript' -def clickhouse_backend_enabled(): - return bool(CLICKHOUSE_SERVICE_HOSTNAME) - - def get_clickhouse_variants(samples, search, user, previous_search_results, genome_version,page=1, num_results=100, sort=None, **kwargs): sample_data_by_dataset_type = _get_sample_data(samples) results = [] @@ -53,7 +49,7 @@ def get_clickhouse_variants(samples, search, user, previous_search_results, geno dataset_results += _evaluate_results(result_q) if has_comp_het: comp_het_sample_data = sample_data - if is_multi_project and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and _is_x_chrom_only(**search): + if is_multi_project and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and _is_x_chrom_only(genome_version, **search): comp_het_sample_data = _no_affected_male_families(sample_data, user) result_q = _get_data_type_comp_het_results_queryset(entry_cls, annotations_cls, comp_het_sample_data, skip_individual_guid=is_multi_project, **search) dataset_results += _evaluate_results(result_q, is_comp_het=True) @@ -272,13 +268,15 @@ def get_transcripts_queryset(genome_version, keys): return TRANSCRIPTS_CLASS_MAP[genome_version].objects.filter(key__in=keys) +def get_transcripts_by_key(genome_version, keys): + return dict(get_transcripts_queryset(genome_version, keys).values_list('key', 'transcripts')) + + def format_clickhouse_results(results, genome_version, **kwargs): keys_with_transcripts = { variant['key'] for result in results for variant in (result if isinstance(result, list) else [result]) if not 'transcripts' in variant } - transcripts_by_key = dict( - get_transcripts_queryset(genome_version, keys_with_transcripts).values_list('key', 'transcripts') - ) + transcripts_by_key = get_transcripts_by_key(genome_version, keys_with_transcripts) formatted_results = [] for variant in results: @@ -388,10 +386,10 @@ def _no_affected_male_families(sample_data, user): } -def _is_x_chrom_only(parsed_locus=None, **kwargs): - parsed_locus = parsed_locus or {} - all_intervals = list((parsed_locus.get('gene_intervals') or {}).values()) + (parsed_locus.get('intervals') or []) - return bool(all_intervals) and all('X' in interval[0] for interval in all_intervals) +def _is_x_chrom_only(genome_version, genes=None, intervals=None, **kwargs): + if not (genes or intervals): + return False + return bool('X' in gene[f'chromGrch{genome_version}'] for gene in (genes or {}).values()) and all('X' in interval['chrom'] for interval in (intervals or [])) OMIM_SORT = 'in_omim' @@ -528,7 +526,7 @@ def _get_sort_key(sort, gene_metadata): return lambda x: tuple(expr(x[0] if isinstance(x, list) else x) for expr in [*sort_expressions, lambda x: x[XPOS_SORT_KEY]]) -def _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples): +def _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples=None): entry_cls = ENTRY_CLASS_MAP[genome_version][data_type] annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][data_type] @@ -539,7 +537,7 @@ def _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples): keys = KEY_LOOKUP_CLASS_MAP[genome_version][data_type].objects.filter(variant_id=variant_id).values_list('key', flat=True) entries = entry_cls.objects.filter(key__in=keys) else: - entries = entry_cls.objects.filter_locus(variant_ids=[variant_id]) + entries = entry_cls.objects.filter_locus(parsed_variant_ids=[variant_id]) entries = entries.result_values(sample_data) results = annotations_cls.objects.subquery_join(entries) @@ -547,17 +545,19 @@ def _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples): # Handles annotation for genotype overrides results = results.filter_annotations(results) else: - results = results.filter_variant_ids(variant_ids=[variant_id]) + results = results.filter_variant_ids(parsed_variant_ids=[variant_id]) variant = results.result_values().first() if variant: variant = format_clickhouse_results([variant], genome_version)[0] return variant -def clickhouse_variant_lookup(user, variant_id, data_type, genome_version=None, samples=None, **kwargs): +def clickhouse_variant_lookup(user, variant_id, dataset_type, sample_type, genome_version): + is_sv = dataset_type == Sample.DATASET_TYPE_SV_CALLS + data_type = f'{dataset_type}_{sample_type}' if is_sv else dataset_type logger.info(f'Looking up variant {variant_id} with data type {data_type}', user) - variant = _clickhouse_variant_lookup(variant_id, genome_version, data_type, samples) + variant = _clickhouse_variant_lookup(variant_id, genome_version, data_type) if variant: _add_liftover_genotypes(variant, data_type, variant_id) else: @@ -567,12 +567,29 @@ def clickhouse_variant_lookup(user, variant_id, data_type, genome_version=None, liftover_results = run_liftover(lifted_genome_version, variant_id[0], variant_id[1]) if liftover_results: lifted_id = (liftover_results[0], liftover_results[1], *variant_id[2:]) - variant = _clickhouse_variant_lookup(lifted_id, lifted_genome_version, data_type, samples) + variant = _clickhouse_variant_lookup(lifted_id, lifted_genome_version, data_type) if not variant: raise ObjectDoesNotExist('Variant not present in seqr') - return variant + variants = [variant] + + if is_sv and variant['svType'] in {'DEL', 'DUP'}: + other_sample_type, other_entry_class = next( + (dt, cls) for dt, cls in ENTRY_CLASS_MAP[genome_version].items() + if dt != data_type and dt.startswith(Sample.DATASET_TYPE_SV_CALLS) + ) + other_annotations_cls = ANNOTATIONS_CLASS_MAP[genome_version][other_sample_type] + + padding = int((variant['end'] - variant['pos']) * 0.2) + entries = other_entry_class.objects.search_padded_interval(variant['chrom'], variant['pos'], padding) + results = other_annotations_cls.objects.subquery_join(entries).search( + padded_interval_end=(variant['end'], padding), + annotations={'structural': [variant['svType'], f"gCNV_{variant['svType']}"]}, + ) + variants += list(results.result_values()) + + return variants def _add_liftover_genotypes(variant, data_type, variant_id): @@ -585,7 +602,7 @@ def _add_liftover_genotypes(variant, data_type, variant_id): ).values_list('key', flat=True) if not keys: return - lifted_entries = lifted_entry_cls.objects.filter_locus(variant_ids=[lifted_id]).filter(key=keys[0]) + lifted_entries = lifted_entry_cls.objects.filter_locus(parsed_variant_ids=[lifted_id]).filter(key=keys[0]) lifted_entry_data = lifted_entries.values('key').annotate( familyGenotypes=GroupArrayArray(lifted_entry_cls.objects.genotype_expression()) ) @@ -647,8 +664,9 @@ def get_clickhouse_keys_for_gene(gene_id, genome_version, dataset_type, keys): def get_all_clickhouse_keys_for_gene(gene_model, genome_version): entry_cls = ENTRY_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS] - return list(entry_cls.objects.filter_locus(require_gene_filter=True, gene_intervals={ - gene_model.id: [getattr(gene_model, f'{field}_grch{genome_version}') for field in ['chrom', 'start', 'end']] + return list(entry_cls.objects.filter_locus(require_gene_filter=True, genes={gene_model.gene_id: { + 'id': gene_model.id, + **{f'{field}Grch{genome_version}': getattr(gene_model, f'{field}_grch{genome_version}') for field in ['chrom', 'start', 'end']}}, }).values_list('key', flat=True).distinct()) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 6ffc0aaaaa..8aa7186072 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -14,11 +14,12 @@ SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4, SV_GENE_COUNTS, NEW_SV_FILTER, GCNV_VARIANT1, GCNV_VARIANT2, \ GCNV_VARIANT3, GCNV_VARIANT4, GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_GENE_COUNTS, \ MULTI_DATA_TYPE_COMP_HET_VARIANT2, ALL_SNV_INDEL_PASS_FILTERS, MULTI_PROJECT_GCNV_VARIANT3, VARIANT_LOOKUP_VARIANT, \ - MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, format_cached_variant + MITO_GENE_COUNTS, PROJECT_4_COMP_HET_VARIANT, SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT_3, \ + format_cached_variant from reference_data.models import Omim from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.utils.search.search_utils_tests import SearchTestHelper -from seqr.utils.search.utils import query_variants, variant_lookup, sv_variant_lookup, get_variant_query_gene_counts, get_single_variant, InvalidSearchException +from seqr.utils.search.utils import query_variants, variant_lookup, get_variant_query_gene_counts, get_single_variant, InvalidSearchException from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin @@ -664,50 +665,45 @@ def test_variant_lookup(self, mock_liftover): mock_convert_coordinate = mock_liftover.return_value.convert_coordinate mock_convert_coordinate.side_effect = lambda chrom, pos: [(chrom, pos + 10000)] - variant = variant_lookup(self.user, ('1', 10439, 'AC', 'A')) - self._assert_expected_variants([variant], [VARIANT_LOOKUP_VARIANT]) + variants = variant_lookup(self.user, '1-10439-AC-A', '38') + self._assert_expected_variants(variants, [VARIANT_LOOKUP_VARIANT]) with self.assertRaises(ObjectDoesNotExist) as cm: - variant_lookup(self.user, ('1', 91511686, 'TCA', 'G')) + variant_lookup(self.user, '1-91511686-TCA-G', '38') self.assertEqual(str(cm.exception), 'Variant not present in seqr') - variant = variant_lookup(self.user, ('7', 143270172, 'A', 'G'), genome_version='37') + variants = variant_lookup(self.user, '7-143270172-A-G', '37') grch37_lookup_variant = { **{k: v for k, v in GRCH37_VARIANT.items() if k not in {'familyGuids', 'genotypes'}}, 'familyGenotypes': {GRCH37_VARIANT['familyGuids'][0]: sorted([ {k: v for k, v in g.items() if k != 'individualGuid'} for g in GRCH37_VARIANT['genotypes'].values() ], key=lambda x: x['sampleId'], reverse=True)}, } - self._assert_expected_variants([variant], [grch37_lookup_variant]) + self._assert_expected_variants(variants, [grch37_lookup_variant]) # Lookup works if variant is only present on a different build - variant = variant_lookup(self.user, ('7', 143260172, 'A', 'G')) - self._assert_expected_variants([variant], [grch37_lookup_variant]) + variants = variant_lookup(self.user, '7-143260172-A-G', '38') + self._assert_expected_variants(variants, [grch37_lookup_variant]) mock_liftover.assert_called_with('hg38', 'hg19') mock_convert_coordinate.assert_called_with('chr7', 143260172) - variant = variant_lookup(self.user, ('M', 4429, 'G', 'A'), genome_version='38') - self._assert_expected_variants([variant], [{ + variants = variant_lookup(self.user, 'M-4429-G-A', '38') + self._assert_expected_variants(variants, [{ **{k: v for k, v in MITO_VARIANT1.items() if k not in {'familyGuids', 'genotypes'}}, 'familyGenotypes': {MITO_VARIANT1['familyGuids'][0]: [ {k: v for k, v in g.items() if k != 'individualGuid'} for g in MITO_VARIANT1['genotypes'].values() ]}, }]) - families = Family.objects.all() - variants = sv_variant_lookup(self.user, 'phase2_DEL_chr14_4640', families, sample_type='WGS') - self._assert_expected_variants(variants, [SV_VARIANT4, GCNV_VARIANT4]) + variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS') + self._assert_expected_variants(variants, [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT]) # reciprocal overlap does not meet the threshold for smaller events - variants = sv_variant_lookup(self.user, 'suffix_140608_DUP', families, sample_type='WES') - self._assert_expected_variants(variants, [GCNV_VARIANT4]) + variants = variant_lookup(self.user, 'suffix_140608_DUP', '38', sample_type='WES') + self._assert_expected_variants(variants, [GCNV_LOOKUP_VARIANT]) - variants = sv_variant_lookup(self.user, 'suffix_140593_DUP', families, sample_type='WES') - self._assert_expected_variants(variants, [GCNV_VARIANT3]) - - with self.assertRaises(ObjectDoesNotExist) as cm: - variant_lookup(self.user, ('5', 143270172, 'A', 'G')) - self.assertEqual(str(cm.exception), 'Variant not present in seqr') + variants = variant_lookup(self.user, 'suffix_140593_DUP', '38', sample_type='WES') + self._assert_expected_variants(variants, [GCNV_LOOKUP_VARIANT_3]) def test_get_single_variant(self): self._set_single_family_search() diff --git a/clickhouse_search/test_utils.py b/clickhouse_search/test_utils.py index 3bfe4c220a..91c001d888 100644 --- a/clickhouse_search/test_utils.py +++ b/clickhouse_search/test_utils.py @@ -1058,8 +1058,40 @@ }], } } +SV_LOOKUP_VARIANT = { + **SV_VARIANT4, + 'familyGenotypes': { + SV_VARIANT4['familyGuids'][0]: [{k: v for k, v in g.items() if k != 'individualGuid'} for g in SV_VARIANT4['genotypes'].values()], + }, +} +GCNV_LOOKUP_VARIANT = { + **GCNV_VARIANT4, + 'familyGenotypes': { + GCNV_VARIANT4['familyGuids'][0]: [{k: v for k, v in g.items() if k != 'individualGuid'} for g in GCNV_VARIANT4['genotypes'].values()], + 'F000002_2_x': [ + {**{k: v for k, v in g.items() if k != 'individualGuid'}, 'familyGuid': 'F000002_2_x'} + for individual_guid, g in GCNV_VARIANT4['genotypes'].items() if individual_guid != 'I000005_hg00732' + ], + }, +} +GCNV_LOOKUP_VARIANT_3 = { + **MULTI_PROJECT_GCNV_VARIANT3, + 'familyGenotypes': { + **{family_guid: [ + {k: v for k, v in g.items() if k != 'individualGuid'} for g in MULTI_PROJECT_GCNV_VARIANT3['genotypes'].values() + if g['familyGuid'] == family_guid] for family_guid in MULTI_PROJECT_GCNV_VARIANT3['familyGuids'] + }, + 'F000002_2_x': [ + {**{k: v for k, v in g.items() if k != 'individualGuid'}, 'familyGuid': 'F000002_2_x'} + for individual_guid, g in GCNV_VARIANT3['genotypes'].items() if individual_guid != 'I000005_hg00732' + ], + }, +} for k in {'familyGuids', 'genotypes'}: VARIANT_LOOKUP_VARIANT.pop(k) + SV_LOOKUP_VARIANT.pop(k) + GCNV_LOOKUP_VARIANT.pop(k) + GCNV_LOOKUP_VARIANT_3.pop(k) PROJECT_4_COMP_HET_VARIANT = { 'key': 22, diff --git a/hail_search/__init__.py b/hail_search/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/__main__.py b/hail_search/__main__.py deleted file mode 100644 index bef783c48a..0000000000 --- a/hail_search/__main__.py +++ /dev/null @@ -1,18 +0,0 @@ -from aiohttp import web -import logging - -from hail_search.web_app import init_web_app - - -def run(): - logging.basicConfig(level=logging.INFO) - app = init_web_app() - web.run_app( - app, - host='0.0.0.0', # nosec - port=5000, - access_log_format='%{From}i "%r" %s %Tfs', - ) - - -run() diff --git a/hail_search/constants.py b/hail_search/constants.py deleted file mode 100644 index 1e4844d7bb..0000000000 --- a/hail_search/constants.py +++ /dev/null @@ -1,91 +0,0 @@ -GENOME_VERSION_GRCh38 = 'GRCh38' -GENOME_VERSION_GRCh37 = 'GRCh37' - -AFFECTED = 'A' -UNAFFECTED = 'N' -UNKNOWN_AFFECTED = 'U' -AFFECTED_ID = 0 -UNAFFECTED_ID = 1 -UNKNOWN_AFFECTED_ID = 2 -AFFECTED_ID_MAP = {AFFECTED: AFFECTED_ID, UNAFFECTED: UNAFFECTED_ID, UNKNOWN_AFFECTED: UNKNOWN_AFFECTED_ID} - -GROUPED_VARIANTS_FIELD = 'variants' -GNOMAD_GENOMES_FIELD = 'gnomad_genomes' -SPLICE_AI_FIELD = 'splice_ai' -NEW_SV_FIELD = 'new_structural_variants' -SCREEN_KEY = 'SCREEN' # uses all caps to match filter provided by the seqr UI -UTR_ANNOTATOR_KEY = 'UTRAnnotator' -EXTENDED_SPLICE_KEY = 'extended_splice_site' -MOTIF_FEATURES_KEY = 'motif_feature' -REGULATORY_FEATURES_KEY = 'regulatory_feature' -CLINVAR_KEY = 'clinvar' -HGMD_KEY = 'hgmd' -STRUCTURAL_ANNOTATION_FIELD = 'structural' -FAMILY_GUID_FIELD = 'familyGuids' -GENOTYPES_FIELD = 'genotypes' - -ANNOTATION_OVERRIDE_FIELDS = [ - SCREEN_KEY, SPLICE_AI_FIELD, NEW_SV_FIELD, STRUCTURAL_ANNOTATION_FIELD, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY, -] -ALLOWED_TRANSCRIPTS = 'allowed_transcripts' -ALLOWED_SECONDARY_TRANSCRIPTS = 'allowed_transcripts_secondary' -HAS_ANNOTATION_OVERRIDE = 'has_annotation_override' -FILTERED_GENE_TRANSCRIPTS = 'gene_transcripts' - -XPOS = 'xpos' - -PATHOGENICTY_SORT_KEY = 'pathogenicity' -PATHOGENICTY_HGMD_SORT_KEY = 'pathogenicity_hgmd' -ABSENT_PATH_SORT_OFFSET = 12.5 -CONSEQUENCE_SORT = 'protein_consequence' -ALPHAMISSENSE_SORT = 'alphamissense' -OMIM_SORT = 'in_omim' - -ALT_ALT = 'alt_alt' -REF_REF = 'ref_ref' -REF_ALT = 'ref_alt' -HAS_ALT = 'has_alt' -HAS_REF = 'has_ref' -COMP_HET_ALT = 'COMP_HET_ALT' - -RECESSIVE = 'recessive' -X_LINKED_RECESSIVE = 'x_linked_recessive' -COMPOUND_HET = 'compound_het' -ANY_AFFECTED = 'any_affected' -RECESSIVE_FILTER = { - AFFECTED_ID: ALT_ALT, - UNAFFECTED_ID: HAS_REF, -} -INHERITANCE_FILTERS = { - RECESSIVE: RECESSIVE_FILTER, - X_LINKED_RECESSIVE: RECESSIVE_FILTER, - 'homozygous_recessive': RECESSIVE_FILTER, - COMPOUND_HET: { - AFFECTED_ID: COMP_HET_ALT, - UNAFFECTED_ID: HAS_REF, - }, - 'de_novo': { - AFFECTED_ID: HAS_ALT, - UNAFFECTED_ID: REF_REF, - }, -} - -PREFILTER_FREQ_CUTOFF = 0.01 -PATH_FREQ_OVERRIDE_CUTOFF = 0.05 -CLINVAR_PATH_FILTER = 'pathogenic' -CLINVAR_LIKELY_PATH_FILTER = 'likely_pathogenic' -CLINVAR_PATH_SIGNIFICANCES = {CLINVAR_PATH_FILTER, CLINVAR_LIKELY_PATH_FILTER} -CLINVAR_PATH_RANGES = [ - (CLINVAR_PATH_FILTER, 'Pathogenic', 'Pathogenic/Likely_risk_allele'), - (CLINVAR_LIKELY_PATH_FILTER, 'Pathogenic/Likely_pathogenic', 'Likely_risk_allele'), - ('vus_or_conflicting', 'Conflicting_classifications_of_pathogenicity', 'No_pathogenic_assertion'), - ('likely_benign', 'Likely_benign', 'Benign/Likely_benign'), - ('benign', 'Benign/Likely_benign', 'Benign'), -] -HGMD_PATH_RANGES = [ - ('disease_causing', 'DM', 'DM'), - ('likely_disease_causing', 'DM?', 'DM?'), - ('hgmd_other', 'DP', None), -] - -MAX_LOAD_INTERVALS = 1000 diff --git a/hail_search/definitions.py b/hail_search/definitions.py deleted file mode 100644 index bc6aa5b204..0000000000 --- a/hail_search/definitions.py +++ /dev/null @@ -1,34 +0,0 @@ -from enum import Enum - - -class SampleType(str, Enum): - WES = 'WES' - WGS = 'WGS' - - @property - def family_entries_field(self) -> str: - return { - SampleType.WES: 'wes_family_entries', - SampleType.WGS: 'wgs_family_entries', - }[self] - - @property - def passes_inheritance_field(self) -> str: - return { - SampleType.WES: 'wes_passes_inheritance', - SampleType.WGS: 'wgs_passes_inheritance', - }[self] - - @property - def passes_quality_field(self) -> str: - return { - SampleType.WES: 'wes_passes_quality', - SampleType.WGS: 'wgs_passes_quality', - }[self] - - @property - def other_sample_type(self) -> 'SampleType': - return { - SampleType.WES: SampleType.WGS, - SampleType.WGS: SampleType.WES, - }[self] diff --git a/hail_search/deploy/Dockerfile b/hail_search/deploy/Dockerfile deleted file mode 100644 index 4a41eb0e7f..0000000000 --- a/hail_search/deploy/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM hailgenetics/hail:0.2.128 - -LABEL maintainer="Broad TGG" - -WORKDIR /hail_search - -# Application Code -COPY hail_search/ . - -WORKDIR / -EXPOSE 5000 -CMD ["python3", "-m", "hail_search"] diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/.README.txt.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/.README.txt.crc deleted file mode 100644 index 11fbf6bc56..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/.metadata.json.gz.crc deleted file mode 100644 index 62b1a3ea46..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/README.txt b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/README.txt deleted file mode 100644 index 5e256b9353..0000000000 --- a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2024/12/05 16:23:46 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/_SUCCESS b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 29578f6349..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/metadata.json.gz deleted file mode 100644 index a7d3fd9f38..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/parts/.part-0.crc deleted file mode 100644 index 6bcbbc2522..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/parts/part-0 deleted file mode 100644 index 6d4137ae10..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/.index.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/.index.crc deleted file mode 100644 index 08fcce0cfd..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/.metadata.json.gz.crc deleted file mode 100644 index 9b8f8c7b50..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/index b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/index deleted file mode 100644 index 39358f3ac9..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/metadata.json.gz deleted file mode 100644 index 48f8e190f2..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/index/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/metadata.json.gz deleted file mode 100644 index 278a872225..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index ba2ccefcd4..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/metadata.json.gz deleted file mode 100644 index 27c89d041d..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/parts/.part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/parts/.part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.crc deleted file mode 100644 index 5a96ef1269..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/parts/.part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/parts/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4 b/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/parts/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4 deleted file mode 100644 index c570d3f221..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/rows/parts/part-0-8dc59bd2-da29-46c0-badd-a77a850af2d4 and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc deleted file mode 100644 index 2abb07dfbc..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc deleted file mode 100644 index 8e6ff63cdd..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/README.txt b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/README.txt deleted file mode 100644 index b41496ec38..0000000000 --- a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2024/06/27 14:14:27 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/_SUCCESS b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index e796fa2531..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz deleted file mode 100644 index 473ee8ea0f..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/.part-0.crc deleted file mode 100644 index 3418d733c1..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/part-0 deleted file mode 100644 index 5beb7ba61a..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.idx/.index.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.idx/.index.crc deleted file mode 100644 index 78fad9791a..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.idx/.metadata.json.gz.crc deleted file mode 100644 index ca274b3389..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.idx/index b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.idx/index deleted file mode 100644 index 3d8c9a969b..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.idx/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.idx/metadata.json.gz deleted file mode 100644 index 14e2c0d67c..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/metadata.json.gz deleted file mode 100644 index eae96f9ae9..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 4a03ffff0c..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/metadata.json.gz deleted file mode 100644 index 02f9d0c657..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/.part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/.part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.crc deleted file mode 100644 index f234dac53f..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/.part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce b/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce deleted file mode 100644 index dbd63db8d8..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/part-0-061e05c1-cd1f-474b-b8f9-472cb004e4ce and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/.README.txt.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/.README.txt.crc deleted file mode 100644 index 17e2f8ae70..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/.metadata.json.gz.crc deleted file mode 100644 index 55d0cdd18d..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/README.txt b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/README.txt deleted file mode 100644 index 22050e815e..0000000000 --- a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2025/01/02 17:23:59 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/_SUCCESS b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 6650687d8b..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/metadata.json.gz deleted file mode 100644 index 95261c2715..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/parts/.part-0.crc deleted file mode 100644 index 871216cce1..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/parts/part-0 deleted file mode 100644 index c33540def9..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/.index.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/.index.crc deleted file mode 100644 index 9e82cfb61e..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/.metadata.json.gz.crc deleted file mode 100644 index e05cfb69de..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/index b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/index deleted file mode 100644 index 0734b2c395..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/metadata.json.gz deleted file mode 100644 index 879fa5f330..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/index/part-0-d213a419-0c77-4952-b924-b6af300f393b.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/metadata.json.gz deleted file mode 100644 index d5dfa64984..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 02e4874e3f..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/metadata.json.gz deleted file mode 100644 index d379d39286..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/parts/.part-0-d213a419-0c77-4952-b924-b6af300f393b.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/parts/.part-0-d213a419-0c77-4952-b924-b6af300f393b.crc deleted file mode 100644 index 433649fc57..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/parts/.part-0-d213a419-0c77-4952-b924-b6af300f393b.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/parts/part-0-d213a419-0c77-4952-b924-b6af300f393b b/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/parts/part-0-d213a419-0c77-4952-b924-b6af300f393b deleted file mode 100644 index a53545dfd8..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/rows/parts/part-0-d213a419-0c77-4952-b924-b6af300f393b and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/.README.txt.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/.README.txt.crc deleted file mode 100644 index eea7b98bfa..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc deleted file mode 100644 index cbb87d8879..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/README.txt b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/README.txt deleted file mode 100644 index 2913b8406f..0000000000 --- a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2024/06/27 14:07:54 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/_SUCCESS b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 4f63d801a1..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/metadata.json.gz deleted file mode 100644 index 1e564200e7..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc deleted file mode 100644 index da922333a3..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/part-0 deleted file mode 100644 index 0dfc12b536..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.idx/.index.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.idx/.index.crc deleted file mode 100644 index 78fad9791a..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.idx/.metadata.json.gz.crc deleted file mode 100644 index ca274b3389..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.idx/index b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.idx/index deleted file mode 100644 index 3d8c9a969b..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.idx/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.idx/metadata.json.gz deleted file mode 100644 index 14e2c0d67c..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/metadata.json.gz deleted file mode 100644 index e43fa94230..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index d00b64c90d..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/metadata.json.gz deleted file mode 100644 index 42b4aee2d2..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/.part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/.part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.crc deleted file mode 100644 index 7078a48348..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/.part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f deleted file mode 100644 index 170603a856..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/part-0-cb4462be-ab9b-4fa8-bc1f-ea8fab6fdf0f and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/.README.txt.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/.README.txt.crc deleted file mode 100644 index d5ad8f7e2d..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/.metadata.json.gz.crc deleted file mode 100644 index cbb87d8879..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/README.txt b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/README.txt deleted file mode 100644 index c7b6eb301a..0000000000 --- a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2025/01/02 17:57:36 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/_SUCCESS b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 52404ebea7..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/metadata.json.gz deleted file mode 100644 index b8223ee0bc..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/parts/.part-0.crc deleted file mode 100644 index 581ceab411..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/parts/part-0 deleted file mode 100644 index 2da408adba..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/.index.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/.index.crc deleted file mode 100644 index 3b5f783b8e..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/.metadata.json.gz.crc deleted file mode 100644 index 8a49d39d80..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/index b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/index deleted file mode 100644 index bf0064cce9..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/metadata.json.gz deleted file mode 100644 index 63e4c6ca64..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/index/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/metadata.json.gz deleted file mode 100644 index e43fa94230..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 3ee859d286..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/metadata.json.gz deleted file mode 100644 index 4b55898614..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/parts/.part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.crc b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/parts/.part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.crc deleted file mode 100644 index 006238b850..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/parts/.part-0-85535ceb-5403-4697-bec1-5eccf7ff958a.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/parts/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a b/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/parts/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a deleted file mode 100644 index a32ae6a5d6..0000000000 Binary files a/hail_search/fixtures/GRCh37/SNV_INDEL/projects/WGS/R0004_non_analyst_project.ht/rows/parts/part-0-85535ceb-5403-4697-bec1-5eccf7ff958a and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/MITO/annotations.ht/.README.txt.crc deleted file mode 100644 index 6487358505..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/annotations.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/MITO/annotations.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/annotations.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/annotations.ht/.metadata.json.gz.crc deleted file mode 100644 index 7683e48825..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/annotations.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/README.txt b/hail_search/fixtures/GRCh38/MITO/annotations.ht/README.txt deleted file mode 100644 index db2cd446be..0000000000 --- a/hail_search/fixtures/GRCh38/MITO/annotations.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.133-4c60fddb171a - Created at 2024/12/04 11:15:26 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/_SUCCESS b/hail_search/fixtures/GRCh38/MITO/annotations.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index f5d5dbdd52..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/metadata.json.gz deleted file mode 100644 index 1987023f59..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/parts/.part-0.crc deleted file mode 100644 index 6e107c2e2c..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/parts/part-0 deleted file mode 100644 index fb90096e10..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/annotations.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/.index.crc b/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/.index.crc deleted file mode 100644 index 60ec208eaa..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/.metadata.json.gz.crc deleted file mode 100644 index 680e9425dc..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/index b/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/index deleted file mode 100644 index b5e11439a9..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/metadata.json.gz deleted file mode 100644 index be26a890c0..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/annotations.ht/index/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/annotations.ht/metadata.json.gz deleted file mode 100644 index 427ed9f6ee..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/annotations.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/annotations.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 4d6a1d646d..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/annotations.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/annotations.ht/rows/metadata.json.gz deleted file mode 100644 index e42080c152..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/annotations.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/rows/parts/.part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.crc b/hail_search/fixtures/GRCh38/MITO/annotations.ht/rows/parts/.part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.crc deleted file mode 100644 index 80a683e7b4..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/annotations.ht/rows/parts/.part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/annotations.ht/rows/parts/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4 b/hail_search/fixtures/GRCh38/MITO/annotations.ht/rows/parts/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4 deleted file mode 100644 index 9510a19286..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/annotations.ht/rows/parts/part-0-5e9d6c60-87bd-401a-a7e9-4cfbc473e5d4 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/.README.txt.crc deleted file mode 100644 index 38887a699a..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/.metadata.json.gz.crc deleted file mode 100644 index 25bea775a4..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/README.txt b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/README.txt deleted file mode 100644 index f3c034c112..0000000000 --- a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.124-13536b531342 - Created at 2023/11/27 16:00:11 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/_SUCCESS b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index b2e1a97f65..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/metadata.json.gz deleted file mode 100644 index aea0d370c4..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/parts/.part-0.crc deleted file mode 100644 index 25c95ae1ef..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/parts/part-0 deleted file mode 100644 index 516777b3f6..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/.index.crc b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/.index.crc deleted file mode 100644 index 1d8418eea4..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/.metadata.json.gz.crc deleted file mode 100644 index 908733b0c2..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/index b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/index deleted file mode 100644 index 719af1fef8..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/metadata.json.gz deleted file mode 100644 index 074fce6abc..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/index/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/metadata.json.gz deleted file mode 100644 index 2a3d11633a..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 10dbb5c451..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/metadata.json.gz deleted file mode 100644 index d2b02cca32..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/parts/.part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.crc b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/parts/.part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.crc deleted file mode 100644 index 276c673d03..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/parts/.part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/parts/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8 b/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/parts/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8 deleted file mode 100644 index 817bc3c3af..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/families/WES/F000002_2.ht/rows/parts/part-0-1db7379b-e75c-4ed2-b79b-28ffb9b115e8 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/MITO/lookup.ht/.README.txt.crc deleted file mode 100644 index 0ab5a311c5..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/lookup.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/MITO/lookup.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/lookup.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/lookup.ht/.metadata.json.gz.crc deleted file mode 100644 index 156c631a1c..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/lookup.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/README.txt b/hail_search/fixtures/GRCh38/MITO/lookup.ht/README.txt deleted file mode 100644 index 2c14db5172..0000000000 --- a/hail_search/fixtures/GRCh38/MITO/lookup.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2024/08/16 15:39:56 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/_SUCCESS b/hail_search/fixtures/GRCh38/MITO/lookup.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/lookup.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 06fd46b58c..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/lookup.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/lookup.ht/globals/metadata.json.gz deleted file mode 100644 index 9d007df8c1..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/lookup.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/MITO/lookup.ht/globals/parts/.part-0.crc deleted file mode 100644 index c62e7c3209..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/lookup.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/MITO/lookup.ht/globals/parts/part-0 deleted file mode 100644 index 94401e711d..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/lookup.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/.index.crc b/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/.index.crc deleted file mode 100644 index c90babdbc4..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/.metadata.json.gz.crc deleted file mode 100644 index 908733b0c2..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/index b/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/index deleted file mode 100644 index 9ccba1bf4b..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/metadata.json.gz deleted file mode 100644 index 074fce6abc..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/lookup.ht/index/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/lookup.ht/metadata.json.gz deleted file mode 100644 index 60381e4539..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/lookup.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 498d4f224f..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/metadata.json.gz deleted file mode 100644 index b54d201997..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/parts/.part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.crc b/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/parts/.part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.crc deleted file mode 100644 index 866aca1166..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/parts/.part-0-87a9f074-c787-4edc-81ce-94ba0daffd80.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/parts/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80 b/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/parts/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80 deleted file mode 100644 index e25b4f60eb..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/lookup.ht/rows/parts/part-0-87a9f074-c787-4edc-81ce-94ba0daffd80 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/.README.txt.crc deleted file mode 100644 index 8d405ad258..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc deleted file mode 100644 index e5c75930c6..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/README.txt b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/README.txt deleted file mode 100644 index 9a2183e75f..0000000000 --- a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2024/04/03 17:42:32 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/_SUCCESS b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 199ff14833..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/metadata.json.gz deleted file mode 100644 index fe5e02236e..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc deleted file mode 100644 index 86bf430a1f..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/parts/part-0 deleted file mode 100644 index fe51e88c40..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/.index.crc b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/.index.crc deleted file mode 100644 index 4a7c7cda60..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/.metadata.json.gz.crc deleted file mode 100644 index 908733b0c2..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/index b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/index deleted file mode 100644 index cb647eb8be..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/metadata.json.gz deleted file mode 100644 index 074fce6abc..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/index/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/metadata.json.gz deleted file mode 100644 index dd915d952f..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index b7a211a5ae..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/metadata.json.gz deleted file mode 100644 index b384d9a465..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/parts/.part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.crc b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/parts/.part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.crc deleted file mode 100644 index 54c1a8dc07..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/parts/.part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/parts/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06 b/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/parts/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06 deleted file mode 100644 index 2ec21b3483..0000000000 Binary files a/hail_search/fixtures/GRCh38/MITO/projects/WES/R0001_1kg.ht/rows/parts/part-0-9a202b26-a04d-4337-9aa5-bbab41b4bc06 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/.README.txt.crc deleted file mode 100644 index 54a0a746a1..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/.metadata.json.gz.crc deleted file mode 100644 index 183ab45fa5..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/README.txt b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/README.txt deleted file mode 100644 index 635c7500c3..0000000000 --- a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2024/03/07 14:19:33 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/_SUCCESS b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 0847bcb1d5..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/metadata.json.gz deleted file mode 100644 index d3f8dc9a63..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/parts/.part-0.crc deleted file mode 100644 index 4b15876391..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/parts/part-0 deleted file mode 100644 index 9bb8bd7ea9..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/index/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.idx/.index.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/index/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.idx/.index.crc deleted file mode 100644 index f3a7f1cf41..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/index/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/index/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/index/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.idx/.metadata.json.gz.crc deleted file mode 100644 index 90992ffff4..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/index/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/index/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.idx/index b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/index/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.idx/index deleted file mode 100644 index 6a49956103..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/index/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/index/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/index/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.idx/metadata.json.gz deleted file mode 100644 index 5342d304f1..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/index/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/metadata.json.gz deleted file mode 100644 index 203b81bb5e..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index eb7f0f681a..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/metadata.json.gz deleted file mode 100644 index 05db6f35fd..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/parts/.part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/parts/.part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.crc deleted file mode 100644 index 2863e352b2..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/parts/.part-0-432b4fda-19b7-4eaf-963f-41f8b545319c.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/parts/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/parts/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c deleted file mode 100644 index db2e04f107..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/rows/parts/part-0-432b4fda-19b7-4eaf-963f-41f8b545319c and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc deleted file mode 100644 index edf58d8f29..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc deleted file mode 100644 index 93463fb756..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/README.txt b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/README.txt deleted file mode 100644 index 6906dbb93f..0000000000 --- a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.124-13536b531342 - Created at 2023/11/21 12:34:02 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/_SUCCESS b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index f51de5e1ea..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz deleted file mode 100644 index 63920a26b3..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/parts/.part-0.crc deleted file mode 100644 index e15f050f66..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/parts/part-0 deleted file mode 100644 index 80e665e3d2..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/index/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.idx/.index.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/index/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.idx/.index.crc deleted file mode 100644 index 1644be3776..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/index/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/index/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/index/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.idx/.metadata.json.gz.crc deleted file mode 100644 index a43f0a0d76..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/index/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/index/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.idx/index b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/index/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.idx/index deleted file mode 100644 index 6369aec8ff..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/index/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/index/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/index/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.idx/metadata.json.gz deleted file mode 100644 index fdd4e325fe..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/index/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/metadata.json.gz deleted file mode 100644 index 50da8a6803..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index dd99e5f775..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/rows/metadata.json.gz deleted file mode 100644 index 3048a5c425..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/rows/parts/.part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.crc b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/rows/parts/.part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.crc deleted file mode 100644 index b09b71de7c..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/rows/parts/.part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/rows/parts/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4 b/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/rows/parts/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4 deleted file mode 100644 index d020604ab8..0000000000 Binary files a/hail_search/fixtures/GRCh38/ONT_SNV_INDEL/families/WES/F000002_2.ht/rows/parts/part-0-7e735aac-e66b-4a34-9b45-5fdd65e9a5b4 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/.README.txt.crc deleted file mode 100644 index f9b6a2f01c..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/.metadata.json.gz.crc deleted file mode 100644 index 16db99e30c..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/README.txt b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/README.txt deleted file mode 100644 index 4483c2b2dc..0000000000 --- a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.133-4c60fddb171a - Created at 2024/12/04 12:35:22 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 65da121be0..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/metadata.json.gz deleted file mode 100644 index 52d10fd556..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/parts/.part-0.crc deleted file mode 100644 index 3e40b058a7..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/parts/part-0 deleted file mode 100644 index 2f32df4a28..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/.index.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/.index.crc deleted file mode 100644 index 765dae3f49..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/.metadata.json.gz.crc deleted file mode 100644 index da6df53be0..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/index b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/index deleted file mode 100644 index 4be95dbd8c..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/metadata.json.gz deleted file mode 100644 index 5b44f36c6f..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/index/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/metadata.json.gz deleted file mode 100644 index 0592867faa..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 8ff1919e93..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/metadata.json.gz deleted file mode 100644 index decb880869..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/parts/.part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/parts/.part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.crc deleted file mode 100644 index 57325f97f8..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/parts/.part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/parts/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3 b/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/parts/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3 deleted file mode 100644 index c2db679b34..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/rows/parts/part-0-2b10c852-02a2-426d-a0cf-961ae060d4c3 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc deleted file mode 100644 index ea27ffe396..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc deleted file mode 100644 index 7260d91792..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/README.txt b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/README.txt deleted file mode 100644 index 04dc59c56a..0000000000 --- a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.124-13536b531342 - Created at 2023/11/27 16:02:25 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 4bef453447..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz deleted file mode 100644 index 9c950612b0..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/.part-0.crc deleted file mode 100644 index 21d561c208..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/part-0 deleted file mode 100644 index 6fd62fa8eb..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/.index.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/.index.crc deleted file mode 100644 index fb65f202b7..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/.metadata.json.gz.crc deleted file mode 100644 index 4eb1dd4f7a..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/index b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/index deleted file mode 100644 index 51f5b8d276..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/metadata.json.gz deleted file mode 100644 index caa522e808..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/index/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/metadata.json.gz deleted file mode 100644 index b423c2cf92..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index f64c31a75f..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/metadata.json.gz deleted file mode 100644 index f36ca2920b..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/.part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/.part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.crc deleted file mode 100644 index 70c55e062c..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/.part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c deleted file mode 100644 index 1827616079..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WES/F000002_2.ht/rows/parts/part-0-5b60e665-6a2b-43ec-b282-1003ad80e87c and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/.README.txt.crc deleted file mode 100644 index 46b5687838..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/.metadata.json.gz.crc deleted file mode 100644 index 63190d7e43..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/README.txt b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/README.txt deleted file mode 100644 index cd3d982528..0000000000 --- a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.133-4c60fddb171a - Created at 2025/01/02 12:26:15 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 4bef453447..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/metadata.json.gz deleted file mode 100644 index 9c950612b0..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/parts/.part-0.crc deleted file mode 100644 index 9944422746..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/parts/part-0 deleted file mode 100644 index 3bc3a51aa6..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/.index.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/.index.crc deleted file mode 100644 index fb65f202b7..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/.metadata.json.gz.crc deleted file mode 100644 index 4eb1dd4f7a..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/index b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/index deleted file mode 100644 index 51f5b8d276..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/metadata.json.gz deleted file mode 100644 index caa522e808..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/index/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/metadata.json.gz deleted file mode 100644 index 3e2c0ff4e9..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index ebb4a5208f..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/metadata.json.gz deleted file mode 100644 index affb24f426..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/parts/.part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/parts/.part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.crc deleted file mode 100644 index ff3efb7411..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/parts/.part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/parts/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd b/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/parts/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd deleted file mode 100644 index e03622879d..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/families/WGS/F000002_2.ht/rows/parts/part-0-428429a4-e72e-454d-a0c4-7bc8ade5d7fd and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/.README.txt.crc deleted file mode 100644 index 92285ec48d..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/.metadata.json.gz.crc deleted file mode 100644 index fc9edcf61d..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/README.txt b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/README.txt deleted file mode 100644 index 5b8a155f43..0000000000 --- a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2024/08/16 15:40:56 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 06fd46b58c..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/metadata.json.gz deleted file mode 100644 index 9d007df8c1..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/parts/.part-0.crc deleted file mode 100644 index 43e325be57..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/parts/part-0 deleted file mode 100644 index 070cc6a220..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/.index.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/.index.crc deleted file mode 100644 index 7013243ff9..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/.metadata.json.gz.crc deleted file mode 100644 index 359650e816..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/index b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/index deleted file mode 100644 index 2196f12697..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/metadata.json.gz deleted file mode 100644 index 521ca22d19..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/index/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/metadata.json.gz deleted file mode 100644 index 50f4a7346a..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index b87073cd48..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/metadata.json.gz deleted file mode 100644 index a63aaa6698..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/parts/.part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/parts/.part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.crc deleted file mode 100644 index 2a7dde15ed..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/parts/.part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/parts/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630 b/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/parts/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630 deleted file mode 100644 index 1d7128c9f7..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/rows/parts/part-0-7a236e4f-7c20-4944-b7d8-071d2b10a630 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/.README.txt.crc deleted file mode 100644 index b214441aa7..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc deleted file mode 100644 index 2cdf8bc33c..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/README.txt b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/README.txt deleted file mode 100644 index 3a80cf1947..0000000000 --- a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.126-ee77707f4fab - Created at 2024/02/05 17:25:57 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index acf12b18f0..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/metadata.json.gz deleted file mode 100644 index a9459e7d89..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc deleted file mode 100644 index db43a2f58b..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/part-0 deleted file mode 100644 index f37af19479..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/.index.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/.index.crc deleted file mode 100644 index 3e56650a62..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/.metadata.json.gz.crc deleted file mode 100644 index baf597cb48..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/index b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/index deleted file mode 100644 index 88a0daa88a..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/metadata.json.gz deleted file mode 100644 index 18bd492643..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/index/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/metadata.json.gz deleted file mode 100644 index 053da1d25e..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index f7f37f8cbc..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/metadata.json.gz deleted file mode 100644 index 5706214db1..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/.part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/.part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.crc deleted file mode 100644 index 8daa7fa73b..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/.part-0-ad3760b2-5a76-4b94-9268-9673bf62e956.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956 b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956 deleted file mode 100644 index e52e118f5c..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0001_1kg.ht/rows/parts/part-0-ad3760b2-5a76-4b94-9268-9673bf62e956 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/.README.txt.crc deleted file mode 100644 index c98011f4e3..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/.metadata.json.gz.crc deleted file mode 100644 index 288ebfdfcf..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/README.txt b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/README.txt deleted file mode 100644 index 0ad192a49b..0000000000 --- a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2024/08/07 16:01:38 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index acf12b18f0..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/metadata.json.gz deleted file mode 100644 index a9459e7d89..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/parts/.part-0.crc deleted file mode 100644 index ec695bda7b..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/parts/part-0 deleted file mode 100644 index adfbcf32cc..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/.index.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/.index.crc deleted file mode 100644 index de6e632532..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/.metadata.json.gz.crc deleted file mode 100644 index ebe47d531c..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/index b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/index deleted file mode 100644 index 55e960e931..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/metadata.json.gz deleted file mode 100644 index 505696c221..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/index/part-0-a35eed71-c848-4567-8937-364bcaecaf47.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/metadata.json.gz deleted file mode 100644 index 9af33596ce..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 78eeac23c3..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/rows/metadata.json.gz deleted file mode 100644 index 457cb5be54..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/rows/parts/.part-0-a35eed71-c848-4567-8937-364bcaecaf47.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/rows/parts/.part-0-a35eed71-c848-4567-8937-364bcaecaf47.crc deleted file mode 100644 index a83168ab2d..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/rows/parts/.part-0-a35eed71-c848-4567-8937-364bcaecaf47.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/rows/parts/part-0-a35eed71-c848-4567-8937-364bcaecaf47 b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/rows/parts/part-0-a35eed71-c848-4567-8937-364bcaecaf47 deleted file mode 100644 index ed2c5b5eed..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WES/R0003_test.ht/rows/parts/part-0-a35eed71-c848-4567-8937-364bcaecaf47 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/.README.txt.crc deleted file mode 100644 index e0d9c28d3d..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/.metadata.json.gz.crc deleted file mode 100644 index e5132ab8ec..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/README.txt b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/README.txt deleted file mode 100644 index 0f10b26898..0000000000 --- a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.126-ee77707f4fab - Created at 2024/02/05 17:29:22 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index acf12b18f0..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/metadata.json.gz deleted file mode 100644 index a9459e7d89..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/parts/.part-0.crc deleted file mode 100644 index 19fddf78f6..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/parts/part-0 deleted file mode 100644 index e5f383002c..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/.index.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/.index.crc deleted file mode 100644 index de6e632532..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/.metadata.json.gz.crc deleted file mode 100644 index ebe47d531c..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/index b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/index deleted file mode 100644 index 55e960e931..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/metadata.json.gz deleted file mode 100644 index 505696c221..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/index/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/metadata.json.gz deleted file mode 100644 index 50f63dd339..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 5679cb0be4..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/metadata.json.gz deleted file mode 100644 index df806dda7b..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/parts/.part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.crc b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/parts/.part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.crc deleted file mode 100644 index a83168ab2d..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/parts/.part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/parts/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d b/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/parts/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d deleted file mode 100644 index ed2c5b5eed..0000000000 Binary files a/hail_search/fixtures/GRCh38/SNV_INDEL/projects/WGS/R0003_test.ht/rows/parts/part-0-28a643dd-8eb0-4510-8718-6e98b4f4274d and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/.README.txt.crc deleted file mode 100644 index 4f4bf2265a..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/.metadata.json.gz.crc deleted file mode 100644 index e1d1584337..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/README.txt b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/README.txt deleted file mode 100644 index c8b92bcc1d..0000000000 --- a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.109-b71b065e4bb6 - Created at 2023/09/08 10:40:43 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 5b3d27b2cb..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/metadata.json.gz deleted file mode 100644 index 13ff2b0cc2..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/parts/.part-0.crc deleted file mode 100644 index 975c25c37e..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/parts/part-0 deleted file mode 100644 index fdfc9d390a..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/.index.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/.index.crc deleted file mode 100644 index 399bfc17c8..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/.metadata.json.gz.crc deleted file mode 100644 index d82735252b..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/index b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/index deleted file mode 100644 index 7f91fffbf2..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/metadata.json.gz deleted file mode 100644 index c951bb1eef..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/index/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/metadata.json.gz deleted file mode 100644 index a76135d54e..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index a99933f31a..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/metadata.json.gz deleted file mode 100644 index 8e91460c4f..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/parts/.part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.crc b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/parts/.part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.crc deleted file mode 100644 index 85cf126ba6..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/parts/.part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/parts/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381 b/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/parts/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381 deleted file mode 100644 index 461ecdebe6..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/annotations.ht/rows/parts/part-0-d065fe2b-41fe-45d2-903c-e0fd3abf9381 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/.README.txt.crc deleted file mode 100644 index 17e76b8f5b..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/.metadata.json.gz.crc deleted file mode 100644 index d51b8848d7..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/README.txt b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/README.txt deleted file mode 100644 index 440bda49da..0000000000 --- a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.109-b71b065e4bb6 - Created at 2023/08/25 10:47:38 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index d2dc39b830..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/metadata.json.gz deleted file mode 100644 index be8cfbcd3f..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/parts/.part-0.crc deleted file mode 100644 index 8f83ae4eea..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/parts/part-0 deleted file mode 100644 index d08c20b476..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/.index.crc b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/.index.crc deleted file mode 100644 index e911752221..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/.metadata.json.gz.crc deleted file mode 100644 index d82735252b..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/index b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/index deleted file mode 100644 index 6ba08ef588..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/metadata.json.gz deleted file mode 100644 index c951bb1eef..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/index/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/metadata.json.gz deleted file mode 100644 index 6a81de8537..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index c05fcd0b44..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/rows/metadata.json.gz deleted file mode 100644 index 72c0c63946..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/rows/parts/.part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.crc b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/rows/parts/.part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.crc deleted file mode 100644 index 07529cf1e0..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/rows/parts/.part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/rows/parts/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48 b/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/rows/parts/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48 deleted file mode 100644 index 0b1e95c86a..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/families/WES/F000002_2.ht/rows/parts/part-0-668-0-0-a8c75457-2bef-dd0d-b9b9-f76af029cc48 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/.README.txt.crc deleted file mode 100644 index cbd98885e5..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc deleted file mode 100644 index c923a35437..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/README.txt b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/README.txt deleted file mode 100644 index dfcd43478a..0000000000 --- a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.126-ee77707f4fab - Created at 2024/02/05 17:32:17 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 79ea0b74d6..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/metadata.json.gz deleted file mode 100644 index c0e3ea8a83..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc deleted file mode 100644 index 85ce8ec7c4..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/parts/part-0 deleted file mode 100644 index de884e1979..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/.index.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/.index.crc deleted file mode 100644 index fe10475d72..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/.metadata.json.gz.crc deleted file mode 100644 index 3bb6392c6f..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/index b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/index deleted file mode 100644 index 19d8ffd823..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/metadata.json.gz deleted file mode 100644 index d86cfffe5f..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/index/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/metadata.json.gz deleted file mode 100644 index 25de203591..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 3a9b95cd93..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/rows/metadata.json.gz deleted file mode 100644 index d757b2db90..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/rows/parts/.part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/rows/parts/.part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.crc deleted file mode 100644 index b3900283d7..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/rows/parts/.part-0-4bb6b390-07db-405c-abad-c57b5aa95da0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/rows/parts/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0 b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/rows/parts/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0 deleted file mode 100644 index 1183fb70d0..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0001_1kg.ht/rows/parts/part-0-4bb6b390-07db-405c-abad-c57b5aa95da0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/.README.txt.crc deleted file mode 100644 index 67686cdbbe..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/.metadata.json.gz.crc deleted file mode 100644 index 69e33ed649..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/README.txt b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/README.txt deleted file mode 100644 index 82a9811b4b..0000000000 --- a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2025/04/24 17:39:28 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 79ea0b74d6..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/metadata.json.gz deleted file mode 100644 index c0e3ea8a83..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/parts/.part-0.crc deleted file mode 100644 index 615c83bb76..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/parts/part-0 deleted file mode 100644 index 37557ad669..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/.index.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/.index.crc deleted file mode 100644 index d69e525c11..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/.metadata.json.gz.crc deleted file mode 100644 index 2ed6ca7df6..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/index b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/index deleted file mode 100644 index 8835e5ade5..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/metadata.json.gz deleted file mode 100644 index edae5a3ff0..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/index/part-0-2c340212-6295-462e-b4f7-e067e4839646.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/metadata.json.gz deleted file mode 100644 index 9eaabdd550..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 8958ab8dce..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/metadata.json.gz deleted file mode 100644 index 2ea2ce5a66..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/parts/.part-0-2c340212-6295-462e-b4f7-e067e4839646.crc b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/parts/.part-0-2c340212-6295-462e-b4f7-e067e4839646.crc deleted file mode 100644 index 97eeca520f..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/parts/.part-0-2c340212-6295-462e-b4f7-e067e4839646.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/parts/part-0-2c340212-6295-462e-b4f7-e067e4839646 b/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/parts/part-0-2c340212-6295-462e-b4f7-e067e4839646 deleted file mode 100644 index f6c59f2434..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WES/projects/WES/R0003_test.ht/rows/parts/part-0-2c340212-6295-462e-b4f7-e067e4839646 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/.README.txt.crc deleted file mode 100644 index 0c85aed7dd..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/.metadata.json.gz.crc deleted file mode 100644 index 093e2169db..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/README.txt b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/README.txt deleted file mode 100644 index b868a31003..0000000000 --- a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.133-4c60fddb171a - Created at 2025/03/07 11:27:39 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 7762d1a08f..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/metadata.json.gz deleted file mode 100644 index ab48b5f481..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/parts/.part-0.crc deleted file mode 100644 index fd4018913c..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/parts/part-0 deleted file mode 100644 index e4ad113e64..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/.index.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/.index.crc deleted file mode 100644 index 4338e8fd27..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/.metadata.json.gz.crc deleted file mode 100644 index 0a1f959fde..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/index b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/index deleted file mode 100644 index a761645d2b..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/metadata.json.gz deleted file mode 100644 index a046bd79e4..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/index/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/metadata.json.gz deleted file mode 100644 index 1c78268552..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index c455749efa..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/metadata.json.gz deleted file mode 100644 index f8b3e06ca3..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/parts/.part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.crc b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/parts/.part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.crc deleted file mode 100644 index b2b31a74fe..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/parts/.part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/parts/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5 b/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/parts/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5 deleted file mode 100644 index d154c9a31d..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/annotations.ht/rows/parts/part-0-fc9f9974-3cb0-4e09-94b8-93bf0d55fbc5 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/.README.txt.crc deleted file mode 100644 index a440037315..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/.metadata.json.gz.crc deleted file mode 100644 index 2817ee0840..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/README.txt b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/README.txt deleted file mode 100644 index cd4126895e..0000000000 --- a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.109-b71b065e4bb6 - Created at 2023/08/23 14:16:34 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 18ad5aa5ab..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/metadata.json.gz deleted file mode 100644 index 19dd3d6727..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/parts/.part-0.crc deleted file mode 100644 index 14b34f102c..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/parts/part-0 deleted file mode 100644 index a6fab3a930..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/index/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.idx/.index.crc b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/index/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.idx/.index.crc deleted file mode 100644 index 644abd8af8..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/index/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/index/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/index/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.idx/.metadata.json.gz.crc deleted file mode 100644 index d47fb2ae41..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/index/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/index/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.idx/index b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/index/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.idx/index deleted file mode 100644 index 02ecf73a76..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/index/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/index/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/index/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.idx/metadata.json.gz deleted file mode 100644 index d6bf3b45f2..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/index/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/metadata.json.gz deleted file mode 100644 index e4b22f3bb4..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 01d66fc815..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/metadata.json.gz deleted file mode 100644 index 1fd6944043..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/parts/.part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.crc b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/parts/.part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.crc deleted file mode 100644 index 0a4059ee39..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/parts/.part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/parts/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003 b/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/parts/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003 deleted file mode 100644 index 3fa93fb57f..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/families/WGS/F000011_11.ht/rows/parts/part-0-278-0-0-7dd50455-5c6c-48a0-7033-11afafa5d003 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/.README.txt.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/.README.txt.crc deleted file mode 100644 index a3bb76f130..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/._SUCCESS.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/.metadata.json.gz.crc deleted file mode 100644 index dfa05f5381..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/README.txt b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/README.txt deleted file mode 100644 index c7d1359830..0000000000 --- a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.126-ee77707f4fab - Created at 2024/02/05 17:34:17 \ No newline at end of file diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/_SUCCESS b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 79ea0b74d6..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/metadata.json.gz deleted file mode 100644 index c0e3ea8a83..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/parts/.part-0.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/parts/.part-0.crc deleted file mode 100644 index 895073fa55..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/parts/part-0 b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/parts/part-0 deleted file mode 100644 index 3a77919b68..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/.index.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/.index.crc deleted file mode 100644 index c196068cba..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/.metadata.json.gz.crc deleted file mode 100644 index 41cee7d3f8..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/index b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/index deleted file mode 100644 index c7ce1f380c..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/index and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/metadata.json.gz deleted file mode 100644 index 67ea06b666..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/index/part-0-cbf84037-3354-427a-98a6-b953711ae5bc.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/metadata.json.gz deleted file mode 100644 index e54270c752..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 59afd18159..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/metadata.json.gz b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/metadata.json.gz deleted file mode 100644 index 49ebd019d1..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/parts/.part-0-cbf84037-3354-427a-98a6-b953711ae5bc.crc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/parts/.part-0-cbf84037-3354-427a-98a6-b953711ae5bc.crc deleted file mode 100644 index 2e373bfe9b..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/parts/.part-0-cbf84037-3354-427a-98a6-b953711ae5bc.crc and /dev/null differ diff --git a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/parts/part-0-cbf84037-3354-427a-98a6-b953711ae5bc b/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/parts/part-0-cbf84037-3354-427a-98a6-b953711ae5bc deleted file mode 100644 index 5455d1fde5..0000000000 Binary files a/hail_search/fixtures/GRCh38/SV_WGS/projects/WGS/R0003_test.ht/rows/parts/part-0-cbf84037-3354-427a-98a6-b953711ae5bc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/.README.txt.crc b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/.README.txt.crc deleted file mode 100644 index 6683baf5ed..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/._SUCCESS.crc b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/.metadata.json.gz.crc deleted file mode 100644 index 152859724c..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/README.txt b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/README.txt deleted file mode 100644 index 0c0b7e087b..0000000000 --- a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.133-4c60fddb171a - Created at 2024/12/04 10:48:02 \ No newline at end of file diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/_SUCCESS b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 889a8c9c90..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/metadata.json.gz deleted file mode 100644 index 04ec6d3b27..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/parts/.part-0.crc b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/parts/.part-0.crc deleted file mode 100644 index 624c959662..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/parts/part-0 b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/parts/part-0 deleted file mode 100644 index e945fc4185..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/.index.crc b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/.index.crc deleted file mode 100644 index 6f23e8d1f3..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/.metadata.json.gz.crc deleted file mode 100644 index 3ed7199fef..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/index b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/index deleted file mode 100644 index 9ac97b7247..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/index and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/metadata.json.gz deleted file mode 100644 index d5cd8343d1..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/index/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/metadata.json.gz deleted file mode 100644 index d2bef9cdc6..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index 2adae799e7..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/metadata.json.gz deleted file mode 100644 index 9e1234feb9..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/parts/.part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.crc b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/parts/.part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.crc deleted file mode 100644 index 3f85bb8ee2..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/parts/.part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/parts/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15 b/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/parts/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15 deleted file mode 100644 index a40e1db48f..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/MITO/clinvar_path_variants.ht/rows/parts/part-0-b856a2ae-7456-4db0-a41f-a4541ce89b15 and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/.README.txt.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/.README.txt.crc deleted file mode 100644 index 562b904ff6..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/._SUCCESS.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/.metadata.json.gz.crc deleted file mode 100644 index 7334a04dc0..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/README.txt b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/README.txt deleted file mode 100644 index a259c4fd70..0000000000 --- a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.133-4c60fddb171a - Created at 2024/12/04 10:46:25 \ No newline at end of file diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/_SUCCESS b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 782d7d3e89..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/metadata.json.gz deleted file mode 100644 index 369b04d91a..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/parts/.part-0.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/parts/.part-0.crc deleted file mode 100644 index 8231b2bcd8..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/parts/part-0 b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/parts/part-0 deleted file mode 100644 index 89e711531d..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/.index.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/.index.crc deleted file mode 100644 index de0c0e4331..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/.metadata.json.gz.crc deleted file mode 100644 index 1ebb404758..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/index b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/index deleted file mode 100644 index c3694ceb09..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/index and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/metadata.json.gz deleted file mode 100644 index 172607a22a..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/index/part-0-c010d599-ce86-4363-be96-0c7b7120788b.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/metadata.json.gz deleted file mode 100644 index b33ebb128c..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index cee57ed241..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/metadata.json.gz deleted file mode 100644 index c487208e68..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/parts/.part-0-c010d599-ce86-4363-be96-0c7b7120788b.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/parts/.part-0-c010d599-ce86-4363-be96-0c7b7120788b.crc deleted file mode 100644 index 4fde886a9f..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/parts/.part-0-c010d599-ce86-4363-be96-0c7b7120788b.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/parts/part-0-c010d599-ce86-4363-be96-0c7b7120788b b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/parts/part-0-c010d599-ce86-4363-be96-0c7b7120788b deleted file mode 100644 index ad1aada680..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/clinvar_path_variants.ht/rows/parts/part-0-c010d599-ce86-4363-be96-0c7b7120788b and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/.README.txt.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/.README.txt.crc deleted file mode 100644 index 2f654d7389..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/.README.txt.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/._SUCCESS.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/._SUCCESS.crc deleted file mode 100644 index 3b7b044936..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/._SUCCESS.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/.metadata.json.gz.crc deleted file mode 100644 index 0373124fc6..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/README.txt b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/README.txt deleted file mode 100644 index 92647e9f70..0000000000 --- a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.133-4c60fddb171a - Created at 2024/12/04 10:46:46 \ No newline at end of file diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/_SUCCESS b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/_SUCCESS deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/.metadata.json.gz.crc deleted file mode 100644 index 782d7d3e89..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/metadata.json.gz deleted file mode 100644 index 369b04d91a..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/parts/.part-0.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/parts/.part-0.crc deleted file mode 100644 index 8231b2bcd8..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/parts/.part-0.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/parts/part-0 b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/parts/part-0 deleted file mode 100644 index 89e711531d..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/globals/parts/part-0 and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/.index.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/.index.crc deleted file mode 100644 index 644f583444..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/.index.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/.metadata.json.gz.crc deleted file mode 100644 index 359650e816..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/index b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/index deleted file mode 100644 index 33d6653b42..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/index and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/metadata.json.gz deleted file mode 100644 index 521ca22d19..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/index/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.idx/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/metadata.json.gz deleted file mode 100644 index 41f6e11f7c..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/.metadata.json.gz.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/.metadata.json.gz.crc deleted file mode 100644 index c844290c9d..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/.metadata.json.gz.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/metadata.json.gz b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/metadata.json.gz deleted file mode 100644 index 8a31cf2f60..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/metadata.json.gz and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/parts/.part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.crc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/parts/.part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.crc deleted file mode 100644 index b8e95019c5..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/parts/.part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc.crc and /dev/null differ diff --git a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/parts/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc b/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/parts/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc deleted file mode 100644 index f43efce500..0000000000 Binary files a/hail_search/fixtures/reference_data/GRCh38/SNV_INDEL/high_af_variants.ht/rows/parts/part-0-40b6b5ff-3831-487d-aa28-ddc09f0c48dc and /dev/null differ diff --git a/hail_search/queries/__init__.py b/hail_search/queries/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hail_search/queries/base.py b/hail_search/queries/base.py deleted file mode 100644 index 518866165a..0000000000 --- a/hail_search/queries/base.py +++ /dev/null @@ -1,1314 +0,0 @@ -from aiohttp.web import HTTPBadRequest, HTTPNotFound, HTTPInternalServerError -from collections import defaultdict, namedtuple -import hail as hl -import logging -import os - -from hail_search.constants import AFFECTED_ID, ALT_ALT, ANNOTATION_OVERRIDE_FIELDS, ANY_AFFECTED, COMP_HET_ALT, \ - COMPOUND_HET, GENOME_VERSION_GRCh38, GROUPED_VARIANTS_FIELD, ALLOWED_TRANSCRIPTS, ALLOWED_SECONDARY_TRANSCRIPTS, HAS_ANNOTATION_OVERRIDE, \ - HAS_ALT, HAS_REF,INHERITANCE_FILTERS, PATH_FREQ_OVERRIDE_CUTOFF, RECESSIVE, REF_ALT, REF_REF, MAX_LOAD_INTERVALS, \ - UNAFFECTED_ID, X_LINKED_RECESSIVE, XPOS, OMIM_SORT, FAMILY_GUID_FIELD, GENOTYPES_FIELD, AFFECTED_ID_MAP, FILTERED_GENE_TRANSCRIPTS - -HAIL_SEARCH_DATA_DIR = os.environ.get('HAIL_SEARCH_DATA_DIR', '/seqr/seqr-hail-search-data') -IN_MEMORY_DIR = os.environ.get('IN_MEMORY_DIR', HAIL_SEARCH_DATA_DIR) - -# Number of filtered genes at which pre-filtering a table by gene-intervals does not improve performance -# Estimated based on behavior for several representative gene lists -MAX_GENE_INTERVALS = int(os.environ.get('MAX_GENE_INTERVALS', MAX_LOAD_INTERVALS)) - -# Optimal number of entry table partitions, balancing parallelization with partition overhead -# Experimentally determined based on compound het search performance: -# https://github.com/broadinstitute/seqr-private/issues/1283#issuecomment-1973392719 -MAX_PARTITIONS = 12 - -logger = logging.getLogger(__name__) - - -PredictionPath = namedtuple('PredictionPath', ['source', 'field', 'format'], defaults=[lambda x: x]) -QualityFilterFormat = namedtuple('QualityFilterFormat', ['scale', 'override'], defaults=[None, None]) - - -def _to_camel_case(snake_case_str): - converted = snake_case_str.replace('_', ' ').title().replace(' ', '') - return converted[0].lower() + converted[1:] - - -class BaseHailTableQuery(object): - - DATA_TYPE = None - KEY_FIELD = None - LOADED_GLOBALS = None - - GENOTYPE_QUERY_MAP = { - REF_REF: lambda gt: gt.is_hom_ref(), - REF_ALT: lambda gt: gt.is_het(), - COMP_HET_ALT: lambda gt: gt.is_het(), - ALT_ALT: lambda gt: gt.is_hom_var(), - HAS_ALT: lambda gt: gt.is_non_ref(), - HAS_REF: lambda gt: gt.is_hom_ref() | gt.is_het_ref(), - } - MISSING_NUM_ALT = -1 - - GENOTYPE_FIELDS = {} - COMPUTED_GENOTYPE_FIELDS = {} - GENOTYPE_OVERRIDE_FIELDS = {} - GENOTYPE_QUERY_FIELDS = {} - QUALITY_FILTER_FORMAT = {} - POPULATIONS = {} - POPULATION_FIELDS = {} - POPULATION_KEYS = ['AF', 'AC', 'AN', 'Hom', 'Hemi', 'Het'] - PREDICTION_FIELDS_CONFIG = {} - ANNOTATION_OVERRIDE_FIELDS = [] - SECONDARY_ANNOTATION_OVERRIDE_FIELDS = None - - GENOME_VERSION = GENOME_VERSION_GRCh38 - GLOBALS = ['enums'] - TRANSCRIPTS_FIELD = None - CORE_FIELDS = [XPOS] - BASE_ANNOTATION_FIELDS = { - FAMILY_GUID_FIELD: lambda r: hl.set( - r.family_entries.filter(hl.is_defined).map(lambda entries: entries.first().familyGuid)), - 'variantId': lambda r: r.variant_id, - } - ENUM_ANNOTATION_FIELDS = { - 'transcripts': { - 'response_key': 'transcripts', - 'empty_array': True, - 'format_array_values': lambda values, *args: values.group_by(lambda t: t.geneId), - }, - } - LIFTOVER_ANNOTATION_FIELDS = { - 'liftedOverGenomeVersion': lambda r: hl.or_missing(hl.is_defined(r.rg37_locus), '37'), - 'liftedOverChrom': lambda r: hl.or_missing(hl.is_defined(r.rg37_locus), r.rg37_locus.contig), - 'liftedOverPos': lambda r: hl.or_missing(hl.is_defined(r.rg37_locus), r.rg37_locus.position), - } - - SORTS = { - XPOS: lambda r: [r.xpos], - 'family_guid': lambda r: [ - hl.int(r.family_entries.find(hl.is_defined).first().familyGuid.first_match_in('(\d+)').first()) - ], - } - - @classmethod - def load_globals(cls): - ht_path = cls._get_table_path('annotations.ht') - try: - ht = hl.read_table(ht_path) - except Exception: - return None - - ht_globals = hl.eval(ht.globals.select(*cls.GLOBALS)) - cls.LOADED_GLOBALS = {k: ht_globals[k] for k in cls.GLOBALS} - return cls.LOADED_GLOBALS - - @classmethod - def _format_population_config(cls, pop_config): - base_pop_config = {field.lower(): field for field in cls.POPULATION_KEYS} - base_pop_config.update(pop_config) - base_pop_config.pop('sort', None) - base_pop_config.pop('sort_subfield', None) - return base_pop_config - - def annotation_fields(self, include_genotype_overrides=True): - annotation_fields = { - GENOTYPES_FIELD: lambda r: r.family_entries.flatmap(lambda x: x).filter( - lambda gt: hl.is_defined(gt.individualGuid) - ).group_by(lambda x: x.individualGuid).map_values(lambda x: self._get_sample_genotype( - x, r, select_fields=['individualGuid'], include_genotype_overrides=include_genotype_overrides, - )), - 'populations': lambda r: hl.struct(**{ - population: self.population_expression(r, population) for population in self.POPULATIONS.keys() - }), - 'predictions': lambda r: hl.struct(**{ - prediction: self._format_enum(r[path.source], path.field, self._enums[path.source][path.field]) - if self._enums.get(path.source, {}).get(path.field) else path.format(r[path.source][path.field]) - for prediction, path in self.PREDICTION_FIELDS_CONFIG.items() - }), - } - annotation_fields.update(self.BASE_ANNOTATION_FIELDS) - annotation_fields.update(self.LIFTOVER_ANNOTATION_FIELDS) - annotation_fields.update(self._additional_annotation_fields()) - - prediction_fields = {path.source for path in self.PREDICTION_FIELDS_CONFIG.values()} - annotation_fields.update([ - self._format_enum_response(k, enum) for k, enum in self._enums.items() - if enum and k not in prediction_fields - ]) - - return annotation_fields - - def _get_sample_genotype(self, samples, r=None, include_genotype_overrides=False, select_fields=None, **kwargs): - sample = samples[0] - return self._select_genotype_for_sample(sample, r, include_genotype_overrides, select_fields) - - def _select_genotype_for_sample(self, sample, r, include_genotype_overrides, select_fields): - return sample.select( - 'sampleId', 'sampleType', 'familyGuid', 'filters', *(select_fields or []), - numAlt=hl.if_else(hl.is_defined(sample.GT), sample.GT.n_alt_alleles(), self.MISSING_NUM_ALT), - **{k: sample[field] for k, field in self.GENOTYPE_FIELDS.items()}, - **{_to_camel_case(k): v(sample, k, r) for k, v in self.COMPUTED_GENOTYPE_FIELDS.items() - if include_genotype_overrides or k not in self.GENOTYPE_OVERRIDE_FIELDS}, - ) - - def _additional_annotation_fields(self): - return {} - - def population_expression(self, r, population): - pop_config = self._format_population_config(self.POPULATIONS[population]) - pop_field = self.POPULATION_FIELDS.get(population, population) - return hl.struct(**{ - response_key: hl.or_else(r[pop_field][field], '' if response_key == 'id' else 0) - for response_key, field in pop_config.items() if field is not None - }) - - def _get_enum_lookup(self, field, subfield, nested_subfield=None): - enum_field = self._enums.get(field, {}) - if subfield: - enum_field = enum_field.get(subfield) - if nested_subfield: - enum_field = enum_field.get(nested_subfield) - if enum_field is None: - return None - return {v: i for i, v in enumerate(enum_field)} - - def _get_enum_terms_ids(self, field, subfield, terms, nested_subfield=None): - if not terms: - return set() - enum = self._get_enum_lookup(field, subfield, nested_subfield=nested_subfield) - return {enum[t] for t in terms if enum.get(t) is not None} - - def _format_enum_response(self, k, enum): - enum_config = self.ENUM_ANNOTATION_FIELDS.get(k, {}) - value = lambda r: self._format_enum(r, k, enum, ht_globals=self._globals, **enum_config) - return enum_config.get('response_key', _to_camel_case(k)), value - - @staticmethod - def _camelcase_value(value): - return value.rename({k: _to_camel_case(k) for k in value.keys()}) - - @classmethod - def _format_enum(cls, r, field, enum, empty_array=False, format_array_values=None, **kwargs): - if hasattr(r, f'{field}_id'): - return hl.array(enum)[r[f'{field}_id']] - - value = r[field] - if hasattr(value, 'map'): - if empty_array: - value = hl.or_else(value, hl.empty_array(value.dtype.element_type)) - value = value.map(lambda x: cls._enum_field(field, x, enum, **kwargs, format_value=cls._camelcase_value)) - if format_array_values: - value = format_array_values(value, r) - return value - - return cls._enum_field(field, value, enum, **kwargs) - - @classmethod - def _enum_field(cls, field_name, value, enum, ht_globals=None, annotate_value=None, format_value=None, drop_fields=None, enum_keys=None, include_version=False, **kwargs): - annotations = {} - drop = [] + (drop_fields or []) - value_keys = value.keys() - for field in (enum_keys or enum.keys()): - field_enum = enum[field] - is_nested_struct = field in value_keys - is_array = f'{field}_ids' in value_keys - - if is_nested_struct: - annotations[field] = cls._enum_field(field, value[field], field_enum, format_value=format_value) - else: - value_field = f"{field}_id{'s' if is_array else ''}" - drop.append(value_field) - enum_array = hl.array(field_enum) - if is_array: - annotations[f'{field}s'] = value[value_field].map(lambda v: enum_array[v]) - else: - annotations[field] = enum_array[value[value_field]] - - if include_version: - annotations['version'] = ht_globals['versions'][field_name] - - value = value.annotate(**annotations) - if annotate_value: - annotations = annotate_value(value, enum) - value = value.annotate(**annotations) - value = value.drop(*drop) - - if format_value: - value = format_value(value) - - return value - - def __init__(self, sample_data, sort=XPOS, sort_metadata=None, num_results=100, inheritance_mode=None, - override_comp_het_alt=False, **kwargs): - if not self.LOADED_GLOBALS: - raise HTTPInternalServerError(reason=f'No loaded {self.DATA_TYPE} data found') - - self.unfiltered_comp_het_ht = None - self._sort = sort - self._sort_metadata = sort_metadata - self._num_results = num_results - self._override_comp_het_alt = override_comp_het_alt - self._ht = None - self._comp_het_ht = None - self._inheritance_mode = inheritance_mode - self._has_secondary_annotations = False - self._is_multi_data_type_comp_het = False - self.max_unaffected_samples = None - self._n_partitions = min(MAX_PARTITIONS, (os.cpu_count() or 2)-1) - self._load_table_kwargs = {'_n_partitions': self._n_partitions} - self.entry_samples_by_family_guid = {} - - if sample_data: - self._load_filtered_table(sample_data, **kwargs) - - @property - def _has_comp_het_search(self): - return self._inheritance_mode in {RECESSIVE, COMPOUND_HET} - - @property - def _globals(self): - return self.LOADED_GLOBALS - - @property - def _enums(self): - return self._globals['enums'] - - def _load_filtered_table(self, sample_data, intervals=None, annotations=None, annotations_secondary=None, overlapped_families=None, **kwargs): - parsed_intervals = self._parse_intervals(intervals, **kwargs) - parsed_annotations = self._parse_annotations(annotations, annotations_secondary, **kwargs) - self.import_filtered_table( - *self._parse_sample_data(sample_data, overlapped_families=overlapped_families), parsed_intervals=parsed_intervals, raw_intervals=intervals, parsed_annotations=parsed_annotations, **kwargs) - - @classmethod - def _get_table_path(cls, path): - return f'{cls._get_table_dir(path)}/{cls.GENOME_VERSION}/{cls.DATA_TYPE}/{path}' - - @classmethod - def _get_table_dir(cls, path): - return IN_MEMORY_DIR if path == 'annotations.ht' else HAIL_SEARCH_DATA_DIR - - def _read_table(self, path, drop_globals=None, skip_missing_field=None): - table_path = self._get_table_path(path) - if 'variant_ht' in self._load_table_kwargs: - ht = self._query_table_annotations(self._load_table_kwargs['variant_ht'], table_path) - if skip_missing_field and not ht.any(hl.is_defined(ht[skip_missing_field])): - return None - ht_globals = hl.read_table(table_path).index_globals() - if drop_globals: - ht_globals = ht_globals.drop(*drop_globals) - return ht.annotate_globals(**ht_globals) - return hl.read_table(table_path, **self._load_table_kwargs) - - @staticmethod - def _query_table_annotations(ht, query_table_path): - query_result = hl.query_table(query_table_path, ht.key).first().drop(*ht.key) - return ht.annotate(**query_result) - - def _parse_sample_data(self, sample_data, overlapped_families=None): - """ - Organizes sample_data by project, sample type, and family in a nested dictionary format. - Returns a tuple containing: - - project_samples (defaultdict): {: {: {: [, ...]}}} - - num_families (int): The number of unique families in the sample data. - """ - if not self._is_multi_data_type_comp_het: - overlapped_families = None - - families = set() - project_samples = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) - for s in sample_data: - if overlapped_families is not None and s['family_guid'] not in overlapped_families: - continue - families.add(s['family_guid']) - project_samples[s['project_guid']][s['sample_type']][s['family_guid']].append(s) - - num_families = len(families) - logger.info(f'Loading {self.DATA_TYPE} data for {num_families} families in {len(project_samples)} projects') - return project_samples, num_families - - def _import_and_filter_multiple_project_hts( - self, project_samples: dict, n_partitions=MAX_PARTITIONS, **kwargs - ) -> tuple[hl.Table, hl.Table]: - """ - In the variant lookup control flow, project_samples looks like this: - {: {: {: True}, : {: True}}, : ...} - In the variant search control flow, project_samples looks like this: - {: {: {: [, , ...]}, : {: []} ...}, : ...} - """ - entries_hts = self._load_project_hts(project_samples, n_partitions, **kwargs) - filtered_project_hts = [] - filtered_comp_het_project_hts = [] - for ht, project_families in entries_hts: - ht, comp_het_ht = self._filter_single_entries_table(ht, project_families, is_merged_ht=True, **kwargs) - if ht is not None: - filtered_project_hts.append(ht) - if comp_het_ht is not None: - filtered_comp_het_project_hts.append(comp_het_ht) - - return self._merge_filtered_hts(filtered_comp_het_project_hts, filtered_project_hts, n_partitions) - - def _load_project_hts(self, project_samples, n_partitions, skip_missing_field=None, **kwargs): - # Need to chunk tables or else evaluating table globals throws LineTooLong exception - # However, minimizing number of chunks minimizes number of aggregations/ evals and improves performance - # Adapted from https://discuss.hail.is/t/importing-many-sample-specific-vcfs/2002/8 - chunk_size = 64 - all_project_hts = [] - project_hts = [] - sample_data = {} - skipped = 0 - - for project_guid, project_sample_type_data in project_samples.items(): - for sample_type, family_sample_data in project_sample_type_data.items(): - project_ht = self._read_project_data(project_guid, sample_type, skip_missing_field=skip_missing_field) - if project_ht is None: - skipped += 1 - continue - project_hts.append(project_ht) - sample_data.update(family_sample_data) - - if len(project_hts) >= chunk_size: - ht = self._prefilter_merged_project_hts(project_hts, n_partitions, **kwargs) - all_project_hts.append((ht, sample_data)) - project_hts = [] - sample_data = {} - - if project_hts: - ht = self._prefilter_merged_project_hts(project_hts, n_partitions, **kwargs) - all_project_hts.append((ht, sample_data)) - - if skipped: - logger.info(f'Skipped {skipped} projects with no matched variants') - - return all_project_hts - - def _import_families_tables(self, project_samples: dict, num_families: int, **kwargs): - if num_families == 1 or len(project_samples) == 1: - project_guid, project_sample_type_data = list(project_samples.items())[0] - return self._import_and_filter_entries_ht( - project_guid, num_families, project_sample_type_data, **kwargs - ) - else: - return self._import_and_filter_multiple_project_hts(project_samples, **kwargs) - - def import_filtered_table(self, project_samples: dict, num_families: int, **kwargs): - families_ht, comp_het_families_ht = self._import_families_tables(project_samples, num_families, **kwargs) - - if comp_het_families_ht is not None: - self._comp_het_ht = self._query_table_annotations(comp_het_families_ht, self._get_table_path('annotations.ht')) - self._comp_het_ht = self._filter_annotated_table(self._comp_het_ht, is_comp_het=True, **kwargs) - self._comp_het_ht = self._filter_compound_hets() - - if families_ht is not None: - self._ht = self._query_table_annotations(families_ht, self._get_table_path('annotations.ht')) - self._ht = self._filter_annotated_table(self._ht, **kwargs) - - def _import_and_filter_entries_ht( - self, project_guid: str, num_families: int, project_sample_type_data, **kwargs - ) -> tuple[hl.Table, hl.Table]: - sample_type = list(project_sample_type_data.keys())[0] - ht, sample_data = self._load_family_or_project_ht( - num_families, project_guid, project_sample_type_data, sample_type, **kwargs - ) - return self._filter_single_entries_table(ht, sample_data, **kwargs) - - def _load_family_or_project_ht(self, num_families, project_guid, project_sample_type_data, sample_type, **kwargs): - if num_families == 1: - family_guid = list(project_sample_type_data[sample_type].keys())[0] - ht, sample_data = self._load_family_ht(family_guid, sample_type, project_sample_type_data, **kwargs) - else: - ht, sample_data = self._load_project_ht(project_guid, sample_type, project_sample_type_data, **kwargs) - return ht, sample_data - - def _load_family_ht( - self, family_guid: str, sample_type: str, project_sample_type_data: dict, **kwargs - ) -> tuple[hl.Table, dict]: - ht = self._read_table(f'families/{sample_type}/{family_guid}.ht') - ht = ht.transmute(family_entries=[ht.entries]) - ht = ht.annotate_globals(family_guids=[family_guid], family_samples={family_guid: ht.sample_ids}) - ht = self._prefilter_entries_table(ht, **kwargs) - sample_data = project_sample_type_data[sample_type] - return ht, sample_data - - def _load_project_ht( - self, project_guid: str, sample_type: str, project_sample_type_data: dict, **kwargs - ) -> tuple[hl.Table, dict]: - ht = self._read_project_table(project_guid, sample_type) - ht = self._prefilter_entries_table(ht, **kwargs) - sample_data = project_sample_type_data[sample_type] - return ht, sample_data - - def _read_project_table(self, project_guid: str, sample_type: str, **kwargs): - return self._read_table(f'projects/{sample_type}/{project_guid}.ht', **kwargs) - - def _read_project_data(self, project_guid: str, sample_type: str, **kwargs): - project_ht = self._read_project_table(project_guid, sample_type, **kwargs) - if project_ht is not None: - project_ht = project_ht.select_globals('sample_type', 'family_guids', 'family_samples') - return project_ht - - def _prefilter_merged_project_hts(self, project_hts, n_partitions, **kwargs): - ht = self._merge_project_hts(project_hts, n_partitions, include_all_globals=True) - return self._prefilter_entries_table(ht, **kwargs) - - @classmethod - def _merge_project_hts(cls, project_hts, n_partitions, include_all_globals=False): - if not project_hts: - return None - ht = hl.Table.multi_way_zip_join(project_hts, 'project_entries', 'project_globals') - ht = ht.repartition(n_partitions) - project_entries = ht.project_entries - if include_all_globals: - project_entries = project_entries.map(cls._annotate_entry_filters) - ht = ht.transmute( - family_entries=hl.enumerate(project_entries).starmap(lambda i, x: hl.or_else( - x.family_entries, - ht.project_globals[i].family_guids.map(lambda f: hl.missing(x.family_entries.dtype.element_type)), - )).flatmap(lambda x: x), - ) - global_expressions = { - 'family_guids': ht.project_globals.flatmap(lambda x: x.family_guids), - } - if include_all_globals: - global_expressions.update({ - 'sample_types': ht.project_globals.flatmap(lambda x: x.family_guids.map(lambda _: x.sample_type)), - 'family_samples': hl.dict(ht.project_globals.flatmap(lambda x: x.family_samples.items())), - }) - - return ht.transmute_globals(**global_expressions) - - def _merge_filtered_hts(self, filtered_comp_het_project_hts, filtered_project_hts, n_partitions): - ht = self._merge_project_hts(filtered_project_hts, n_partitions) - comp_het_ht = self._merge_project_hts(filtered_comp_het_project_hts, n_partitions) - return ht, comp_het_ht - - @staticmethod - def _apply_entry_filters(ht, select_globals=True): - if ht is not None: - ht = ht.filter(ht.family_entries.any(hl.is_defined)) - if select_globals: - ht = ht.select_globals('family_guids') - return ht - - def _filter_single_entries_table(self, ht, project_families, inheritance_filter=None, quality_filter=None, is_merged_ht=False, **kwargs): - ht, sorted_family_sample_data = self._add_entry_sample_families(ht, project_families, is_merged_ht) - ht = self._filter_quality(ht, quality_filter, **kwargs) - ht, ch_ht = self._filter_inheritance(ht, None, inheritance_filter, sorted_family_sample_data) - ht = self._apply_entry_filters(ht) - ch_ht = self._apply_entry_filters(ch_ht) - - return ht, ch_ht - - def _filter_quality( - self, ht, quality_filter, annotation='family_entries', entries_ht_field='family_entries', **kwargs - ): - passes_quality_filter = self._get_family_passes_quality_filter( - quality_filter, ht, **kwargs - ) - ht_entries = ht[entries_ht_field] - - if passes_quality_filter is not None: - return ht.annotate(**{ - annotation: ht_entries.map( - lambda entries: hl.or_missing(passes_quality_filter(entries), entries) - )}) - - return ht.annotate(**{annotation: ht_entries}) - - def _add_entry_sample_families(self, ht, sample_data, is_merged_ht): - """ - Annotates samples in family_entries with additional sample-level data. - returns a tuple containing: - - ht (hl.Table) - - sorted_family_sample_data (list): A list of lists containing sample data for each family, - sorted in the same order as is in family_entries. [[, ]] - """ - ht_globals = hl.eval(ht.globals) - - missing_samples = set() - family_sample_index_data = [] - sorted_family_sample_data = [] - family_guids = sorted(sample_data.keys()) - for family_guid in family_guids: - ht_family_samples = ht_globals.family_samples[family_guid] - samples = sample_data[family_guid] - if samples is True: - samples = ht_family_samples - get_sample_data = lambda s: {'sampleId': s} - missing_family_samples = [] - else: - get_sample_data = self._sample_entry_data - missing_family_samples = [s['sample_id'] for s in samples if s['sample_id'] not in ht_family_samples] - if missing_family_samples: - missing_samples.update(missing_family_samples) - else: - family_index = ht_globals.family_guids.index(family_guid) - family_entry_data = { - 'sampleType': self._get_sample_type(family_index, ht_globals), - 'familyGuid': family_guid, - } - formatted_samples = [{**family_entry_data, **get_sample_data(s)} for s in samples] - sample_index_data = [(ht_family_samples.index(s['sampleId']), hl.struct(**s)) for s in formatted_samples] - family_sample_index_data.append((family_index, sample_index_data)) - sorted_family_sample_data.append(formatted_samples) - self.entry_samples_by_family_guid[family_guid] = [s['sampleId'] for s in formatted_samples] - - if missing_samples: - raise HTTPBadRequest( - reason=f'The following samples are available in seqr but missing the loaded data: {", ".join(sorted(missing_samples))}' - ) - - family_sample_index_data = hl.array(family_sample_index_data) - - ht = ht.annotate_globals(family_guids=family_guids) - - ht = ht.transmute(family_entries=family_sample_index_data.map(lambda family_tuple: family_tuple[1].map( - lambda sample_tuple: ht.family_entries[family_tuple[0]][sample_tuple[0]].annotate(**sample_tuple[1]) - ))) - if not is_merged_ht: - ht = self._annotate_entry_filters(ht) - - return ht, sorted_family_sample_data - - @staticmethod - def _annotate_entry_filters(ht): - return ht.annotate(family_entries=ht.family_entries.map( - lambda entries: entries.map(lambda x: x.annotate(filters=ht.filters)) - )) - - @classmethod - def _sample_entry_data(cls, sample): - return dict( - sampleId=sample['sample_id'], - individualGuid=sample['individual_guid'], - affected_id=AFFECTED_ID_MAP.get(sample['affected']), - is_male=sample.get('is_male', False), - ) - - @classmethod - def _get_sample_type(cls, family_index, ht_globals): - if 'sample_types' in ht_globals: - return ht_globals.sample_types[family_index] - return ht_globals.sample_type - - def _filter_inheritance( - self, ht, comp_het_ht, inheritance_filter, sorted_family_sample_data, - annotation='family_entries', entries_ht_field='family_entries', **kwargs - ): - any_valid_entry = lambda x: self.GENOTYPE_QUERY_MAP[HAS_ALT](x.GT) - - is_any_affected = self._inheritance_mode == ANY_AFFECTED - if is_any_affected: - prev_any_valid_entry = any_valid_entry - any_valid_entry = lambda x: prev_any_valid_entry(x) & (x.affected_id == AFFECTED_ID) - - ht = ht.annotate(**{ - entries_ht_field: ht[entries_ht_field].map( - lambda entries: hl.or_missing(entries.any(any_valid_entry), entries) - )}) - - if self._has_comp_het_search: - comp_het_ht = self._annotate_families_inheritance( - comp_het_ht if comp_het_ht is not None else ht, COMPOUND_HET, inheritance_filter, - sorted_family_sample_data, annotation, entries_ht_field, **kwargs - ) - - if is_any_affected or not (inheritance_filter or self._inheritance_mode): - # No sample-specific inheritance filtering needed - sorted_family_sample_data = [] - - ht = None if self._inheritance_mode == COMPOUND_HET else self._annotate_families_inheritance( - ht, self._inheritance_mode, inheritance_filter, sorted_family_sample_data, - annotation, entries_ht_field, **kwargs - ) - - return ht, comp_het_ht - - def _annotate_families_inheritance( - self, ht, inheritance_mode, inheritance_filter, sorted_family_sample_data, - annotation, entries_ht_field, family_passes_inheritance_filter = None - ): - if not family_passes_inheritance_filter: - family_passes_inheritance_filter = self._get_family_passes_inheritance_filter - - entry_indices_by_gt = self._get_entry_indices_by_gt_map( - inheritance_filter, inheritance_mode, sorted_family_sample_data - ) - - for genotype, entry_indices in entry_indices_by_gt.items(): - if not entry_indices: - continue - entry_indices = hl.dict(entry_indices) - ht = ht.annotate(**{ - annotation: hl.enumerate(ht[entries_ht_field]).starmap( - lambda family_idx, family_samples: family_passes_inheritance_filter( - entry_indices, family_idx, genotype, family_samples, ht, annotation - ) - ) - }) - - return ht - - def _get_family_passes_inheritance_filter(self, entry_indices, family_idx, genotype, family_samples, *args): - return hl.or_missing( - ~entry_indices.contains(family_idx) | entry_indices[family_idx].all( - lambda sample_i: self.GENOTYPE_QUERY_MAP[genotype](family_samples[sample_i].GT) - ), family_samples) - - def _get_entry_indices_by_gt_map(self, inheritance_filter, inheritance_mode, sorted_family_sample_data): - individual_genotype_filter = (inheritance_filter or {}).get('genotype') - - # Create a mapping of genotypes to check against a list of samples for a family - entry_indices_by_gt = defaultdict(lambda: defaultdict(list)) - for family_index, samples in enumerate(sorted_family_sample_data): - for sample_index, s in enumerate(samples): - genotype = individual_genotype_filter.get(s['individualGuid']) \ - if individual_genotype_filter else INHERITANCE_FILTERS[inheritance_mode].get(s['affected_id']) - if inheritance_mode == X_LINKED_RECESSIVE and s['affected_id'] == UNAFFECTED_ID and s['is_male']: - genotype = REF_REF - if genotype == COMP_HET_ALT and self._override_comp_het_alt: - genotype = HAS_ALT - if genotype: - entry_indices_by_gt[genotype][family_index].append(sample_index) - - if inheritance_mode == COMPOUND_HET: - family_unaffected_counts = [ - len(i) for i in entry_indices_by_gt[INHERITANCE_FILTERS[COMPOUND_HET][UNAFFECTED_ID]].values() - ] - self.max_unaffected_samples = max(family_unaffected_counts) if family_unaffected_counts else 0 - - return entry_indices_by_gt - - def _get_family_passes_quality_filter(self, quality_filter, ht, **kwargs): - quality_filter = quality_filter or {} - - affected_only = quality_filter.get('affected_only') - passes_quality_filters = [] - for filter_k, value in quality_filter.items(): - genotype_key = filter_k.replace('min_', '') - field = self.GENOTYPE_QUERY_FIELDS.get(genotype_key, self.GENOTYPE_FIELDS.get(genotype_key)) - if field and value: - passes_quality_filters.append(self._get_genotype_passes_quality_field(field, value, affected_only)) - - if quality_filter.get('vcf_filter'): - passes_quality_filters.append(self._passes_vcf_filters) - - if not passes_quality_filters: - return None - - return lambda entries: entries.all(lambda gt: hl.all([f(gt) for f in passes_quality_filters])) - - @classmethod - def _get_genotype_passes_quality_field(cls, field, value, affected_only, **kwargs): - field_config = cls.QUALITY_FILTER_FORMAT.get(field) or QualityFilterFormat() - if field_config.scale: - value = value / field_config.scale - - def passes_quality_field(gt): - is_valid = (gt[field] >= value) | hl.is_missing(gt[field]) - if field_config.override: - is_valid |= field_config.override(gt) - if affected_only: - is_valid |= gt.affected_id != AFFECTED_ID - return is_valid - - return passes_quality_field - - @staticmethod - def _passes_vcf_filters(gt): - return hl.is_missing(gt.filters) | (gt.filters.length() < 1) - - def _parse_variant_keys(self, variant_keys): - return [hl.struct(**{self.KEY_FIELD[0]: key}) for key in (variant_keys or [])] - - def _prefilter_entries_table(self, ht, **kwargs): - return ht - - def _filter_annotated_table(self, ht, gene_ids=None, rs_ids=None, frequencies=None, in_silico=None, pathogenicity=None, - parsed_annotations=None, is_comp_het=False, exclude=None, **kwargs): - - ht = self._filter_excluded(ht, exclude) - - ht = self._filter_by_gene_ids(ht, hl.set(gene_ids) if gene_ids else None) - - if rs_ids: - ht = self._filter_rs_ids(ht, rs_ids) - - ht = self._filter_by_frequency(ht, frequencies, pathogenicity) - - ht = self._filter_by_in_silico(ht, in_silico) - - return self._filter_by_annotations(ht, is_comp_het=is_comp_het, **(parsed_annotations or {})) - - @classmethod - def _annotate_filtered_gene_transcripts(cls, ht, gene_ids): - return ht.annotate( - **{FILTERED_GENE_TRANSCRIPTS: ht[cls.TRANSCRIPTS_FIELD].filter(lambda t: gene_ids.contains(t.gene_id))} - ) - - def _filter_excluded(self, ht, exclude): - return ht - - def _filter_by_gene_ids(self, ht, gene_ids): - if gene_ids is None: - return ht - - ht = self._annotate_filtered_gene_transcripts(ht, gene_ids) - return ht.filter(hl.is_defined(ht[FILTERED_GENE_TRANSCRIPTS].first())) - - def _filter_rs_ids(self, ht, rs_ids): - rs_id_set = hl.set(rs_ids) - return ht.filter(rs_id_set.contains(ht.rsid)) - - def _parse_intervals(self, intervals, gene_ids=None, variant_keys=None, variant_ids=None, exclude_intervals=False, **kwargs): - parsed_variant_keys = self._parse_variant_keys(variant_keys) - if parsed_variant_keys: - self._load_table_kwargs['variant_ht'] = hl.Table.parallelize(parsed_variant_keys).key_by(*self.KEY_FIELD) - return intervals - - if variant_ids: - first_chrom = variant_ids[0][0] - if all(first_chrom == v[0] for v in variant_ids): - positions = [pos for _, pos, _, _ in variant_ids] - intervals = [(first_chrom, min(positions), max(positions) + 1)] - else: - intervals = [(chrom, pos, pos+1) for chrom, pos, _, _ in variant_ids] - - is_x_linked = self._inheritance_mode == X_LINKED_RECESSIVE - if not (intervals or is_x_linked): - return intervals - - raw_intervals = intervals - if self._should_add_chr_prefix(): - intervals = [[f'chr{interval[0]}', *interval[1:]] for interval in (intervals or [])] - - if len(intervals) > MAX_GENE_INTERVALS and len(intervals) == len(gene_ids or []): - intervals = self.cluster_intervals(sorted(intervals)) - - parsed_intervals = [ - hl.eval(hl.locus_interval(*interval, reference_genome=self.GENOME_VERSION, invalid_missing=True)) - for interval in (intervals or []) - ] - invalid_intervals = [raw_intervals[i] for i, interval in enumerate(parsed_intervals) if interval is None] - if invalid_intervals: - error_interval = ', '.join([f'{chrom}:{start}-{end}' for chrom, start, end in invalid_intervals]) - raise HTTPBadRequest(reason=f'Invalid intervals: {error_interval}') - - if is_x_linked: - reference_genome = hl.get_reference(self.GENOME_VERSION) - parsed_intervals.append( - hl.eval(hl.parse_locus_interval(reference_genome.x_contigs[0], reference_genome=self.GENOME_VERSION)) - ) - - if parsed_intervals and not exclude_intervals: - self._set_interval_prefilter(parsed_intervals) - - return parsed_intervals - - def _set_interval_prefilter(self, parsed_intervals): - if len(parsed_intervals) < MAX_LOAD_INTERVALS: - self._load_table_kwargs = {'_intervals': parsed_intervals, '_filter_intervals': True} - - @classmethod - def cluster_intervals(cls, intervals, distance=100000, max_intervals=MAX_GENE_INTERVALS): - if len(intervals) <= max_intervals: - return intervals - - merged_intervals = [intervals[0]] - for chrom, start, end in intervals[1:]: - prev_chrom, prev_start, prev_end = merged_intervals[-1] - if chrom == prev_chrom and start - prev_end < distance: - merged_intervals[-1] = [chrom, prev_start, max(prev_end, end)] - else: - merged_intervals.append([chrom, start, end]) - - return cls.cluster_intervals(merged_intervals, distance=distance+100000, max_intervals=max_intervals) - - def _should_add_chr_prefix(self): - return self.GENOME_VERSION == GENOME_VERSION_GRCh38 - - def _filter_by_frequency(self, ht, frequencies, pathogenicity): - frequencies = {k: v for k, v in (frequencies or {}).items() if k in self.POPULATIONS} - if not frequencies: - return ht - - path_override_filter = self._frequency_override_filter(ht, pathogenicity) - filters = [] - for pop, freqs in sorted(frequencies.items()): - pop_filters = [] - pop_expr = ht[self.POPULATION_FIELDS.get(pop, pop)] - pop_config = self._format_population_config(self.POPULATIONS[pop]) - if freqs.get('af') is not None and freqs['af'] < 1: - af_field = pop_config.get('filter_af') or pop_config['af'] - if af_field is None: - continue - pop_filter = pop_expr[af_field] <= freqs['af'] - if path_override_filter is not None and freqs['af'] < PATH_FREQ_OVERRIDE_CUTOFF: - pop_filter |= path_override_filter & (pop_expr[af_field] <= PATH_FREQ_OVERRIDE_CUTOFF) - pop_filters.append(pop_filter) - elif freqs.get('ac') is not None: - ac_field = pop_config['ac'] - if ac_field: - pop_filters.append(pop_expr[ac_field] <= freqs['ac']) - - if freqs.get('hh') is not None: - for field_name in ['hom', 'hemi']: - field = pop_config[field_name] - if field: - pop_filter = pop_expr[field] <= freqs['hh'] - if path_override_filter is not None: - pop_filter |= path_override_filter - pop_filters.append(pop_filter) - - if pop_filters: - filters.append(hl.is_missing(pop_expr) | hl.all(pop_filters)) - - if filters: - ht = ht.filter(hl.all(filters)) - return ht - - def _frequency_override_filter(self, ht, pathogenicity): - return None - - def _filter_by_in_silico(self, ht, in_silico_filters): - in_silico_filters = in_silico_filters or {} - require_score = in_silico_filters.get('requireScore', False) - in_silico_filters = {k: v for k, v in in_silico_filters.items() if k in self.PREDICTION_FIELDS_CONFIG and v} - if not in_silico_filters: - return ht - - in_silico_qs = [] - missing_qs = [] - for in_silico, value in in_silico_filters.items(): - score_filter, ht_value = self._get_in_silico_filter(ht, in_silico, value) - in_silico_qs.append(score_filter) - if not require_score: - missing_qs.append(hl.is_missing(ht_value)) - - if missing_qs: - in_silico_qs.append(hl.all(missing_qs)) - - return ht.filter(hl.any(in_silico_qs)) - - def _get_in_silico_filter(self, ht, in_silico, value): - score_path = self.PREDICTION_FIELDS_CONFIG[in_silico] - enum_lookup = self._get_enum_lookup(*score_path[:2]) - if enum_lookup is not None: - ht_value = ht[score_path.source][f'{score_path.field}_id'] - score_filter = ht_value == enum_lookup[value] - else: - ht_value = ht[score_path.source][score_path.field] - score_filter = ht_value >= float(value) - - return score_filter, ht_value - - def _parse_annotations(self, annotations, annotations_secondary, **kwargs): - annotations = annotations or {} - allowed_consequence_ids = self._get_allowed_consequence_ids(annotations) - annotation_overrides = self._get_annotation_override_fields(annotations, **kwargs) - - parsed_annotations = {} - if self._has_comp_het_search and annotations_secondary: - secondary_allowed_consequence_ids = self._get_allowed_consequence_ids(annotations_secondary) - has_different_secondary = secondary_allowed_consequence_ids != allowed_consequence_ids - has_data_type_primary_annotations = allowed_consequence_ids or annotation_overrides - has_data_type_secondary_annotations = bool(secondary_allowed_consequence_ids) - secondary_annotation_overrides = None - if self.SECONDARY_ANNOTATION_OVERRIDE_FIELDS: - secondary_annotation_overrides = self._get_annotation_override_fields( - annotations_secondary, override_fields=self.SECONDARY_ANNOTATION_OVERRIDE_FIELDS) - has_data_type_secondary_annotations |= bool(secondary_annotation_overrides) - has_different_secondary |= secondary_annotation_overrides != annotation_overrides - - if not has_data_type_primary_annotations: - allowed_consequence_ids = secondary_allowed_consequence_ids - annotation_overrides = secondary_annotation_overrides - # Data type only has annotations for second hit, so there is no need for the homozygous recessive query - self._inheritance_mode = COMPOUND_HET - elif has_different_secondary: - parsed_annotations.update({ - 'secondary_consequence_ids': secondary_allowed_consequence_ids, - 'secondary_annotation_overrides': secondary_annotation_overrides, - }) - self._has_secondary_annotations = True - - if not (has_data_type_primary_annotations and has_data_type_secondary_annotations): - self._is_multi_data_type_comp_het = True - - parsed_annotations.update({ - 'consequence_ids': allowed_consequence_ids, - 'annotation_overrides': annotation_overrides, - }) - return parsed_annotations - - def _filter_by_annotations(self, ht, is_comp_het=False, consequence_ids=None, annotation_overrides=None, - secondary_consequence_ids=None, secondary_annotation_overrides=None, **kwargs): - - annotation_exprs = {} - if consequence_ids: - annotation_exprs[ALLOWED_TRANSCRIPTS] = self._get_allowed_transcripts(ht, consequence_ids) - ht = ht.annotate(**annotation_exprs) - annotation_filters = self._get_annotation_override_filters(ht, annotation_overrides or {}) - - if not is_comp_het: - if annotation_exprs: - annotation_filters.append(self._has_allowed_transcript_filter(ht, ALLOWED_TRANSCRIPTS)) - if annotation_filters: - ht = ht.filter(hl.any(annotation_filters)) - return ht - - if annotation_filters: - annotation_exprs[HAS_ANNOTATION_OVERRIDE] = hl.any(annotation_filters) - if secondary_annotation_overrides is not None: - overrides = self._get_annotation_override_filters(ht, secondary_annotation_overrides) - annotation_exprs[f'{HAS_ANNOTATION_OVERRIDE}_secondary'] = hl.any(overrides) if overrides else False - secondary_allowed_transcripts_field = f'{ALLOWED_TRANSCRIPTS}_secondary' - if secondary_consequence_ids: - annotation_exprs[secondary_allowed_transcripts_field] = self._get_allowed_transcripts(ht, secondary_consequence_ids) - - filter_fields = list(annotation_exprs.keys()) - transcript_filter_fields = [ALLOWED_TRANSCRIPTS, secondary_allowed_transcripts_field] - annotation_exprs.pop(ALLOWED_TRANSCRIPTS, None) - if annotation_exprs: - ht = ht.annotate(**annotation_exprs) - - all_filters = [ - self._has_allowed_transcript_filter(ht, field) if field in transcript_filter_fields else ht[field] - for field in filter_fields - ] - return ht.filter(hl.any(all_filters)) - - def _get_allowed_consequence_ids(self, annotations): - allowed_consequences = { - ann for field, anns in annotations.items() - if anns and (field not in ANNOTATION_OVERRIDE_FIELDS) for ann in anns - } - return self._get_enum_terms_ids(self.TRANSCRIPTS_FIELD, self.TRANSCRIPT_CONSEQUENCE_FIELD, allowed_consequences) - - def _get_allowed_transcripts(self, ht, allowed_consequence_ids): - transcript_filter = self._get_allowed_transcripts_filter(allowed_consequence_ids) - return getattr(ht, FILTERED_GENE_TRANSCRIPTS, ht[self.TRANSCRIPTS_FIELD]).filter(transcript_filter) - - @staticmethod - def _get_allowed_transcripts_filter(allowed_consequence_ids): - allowed_consequence_ids = hl.set(allowed_consequence_ids) - return lambda gc: allowed_consequence_ids.contains(gc.major_consequence_id) - - def _get_annotation_override_fields(self, annotations, override_fields=None, **kwargs): - override_fields = override_fields or self.ANNOTATION_OVERRIDE_FIELDS - return {k: annotations[k] for k in override_fields if annotations.get(k)} - - def _get_annotation_override_filters(self, ht, annotation_overrides): - return [] - - def _get_annotation_filters(self, ht, is_secondary=False): - suffix = '_secondary' if is_secondary else '' - annotation_filters = [] - - allowed_transcripts_field = f'{ALLOWED_TRANSCRIPTS}{suffix}' - if allowed_transcripts_field in ht.row: - annotation_filters.append(self._has_allowed_transcript_filter(ht, allowed_transcripts_field)) - - annotation_override_field = f'{HAS_ANNOTATION_OVERRIDE}{suffix}' - if annotation_override_field in ht.row: - annotation_filters.append(ht[annotation_override_field]) - elif HAS_ANNOTATION_OVERRIDE in ht.row: - # For secondary annotations, if no secondary override is defined use the primary override - annotation_filters.append(ht[HAS_ANNOTATION_OVERRIDE]) - - return annotation_filters - - @staticmethod - def _has_allowed_transcript_filter(ht, allowed_transcript_field): - return hl.is_defined(ht[allowed_transcript_field].first()) - - def _filter_compound_hets(self): - # pylint: disable=pointless-string-statement - ch_ht = self._comp_het_ht - - # Get possible pairs of variants within the same gene - def key(v): - ks = [v[k] for k in self.KEY_FIELD] - return ks[0] if len(self.KEY_FIELD) == 1 else hl.tuple(ks) - ch_ht = ch_ht.annotate(key_=key(ch_ht.row), gene_ids=self._gene_ids_expr(ch_ht, filtered_genes_only=True)) - ch_ht = ch_ht.explode(ch_ht.gene_ids) - - # Filter allowed transcripts to the grouped gene - transcript_annotations = { - k: ch_ht[k].filter(lambda t: t.gene_id == ch_ht.gene_ids) - for k in [ALLOWED_TRANSCRIPTS, ALLOWED_SECONDARY_TRANSCRIPTS] if k in ch_ht.row - } - if transcript_annotations: - ch_ht = ch_ht.annotate(**transcript_annotations) - - if transcript_annotations or self._has_secondary_annotations: - primary_filters = self._get_annotation_filters(ch_ht) - ch_ht = ch_ht.annotate(is_primary=hl.coalesce(hl.any(primary_filters), False)) - if self._has_secondary_annotations and not self._is_multi_data_type_comp_het: - secondary_filters = self._get_annotation_filters(ch_ht, is_secondary=True) - is_secondary = hl.coalesce(hl.any(secondary_filters), False) - else: - is_secondary = ch_ht.is_primary - ch_ht = ch_ht.annotate(is_secondary=is_secondary) - if transcript_annotations: - ch_ht = ch_ht.filter(ch_ht.is_primary | ch_ht.is_secondary) - else: - ch_ht = ch_ht.annotate(is_primary=True, is_secondary=True) - self.unfiltered_comp_het_ht = ch_ht - - if self._is_multi_data_type_comp_het: - # In cases where comp het pairs must have different data types, there are no single data type results - return None - - """ Assume a table with the following data - key_ | gene_ids | is_primary | is_secondary - 1 | A | true | true - 2 | A | true | true - 2 | B | true | true - 3 | A | false | true - 3 | B | false | true - """ - - variants = ch_ht.collect(_localize=False) - variants = variants.group_by(lambda v: v.gene_ids) - variants = variants.items().flatmap(lambda gvs: - hl.rbind(gvs[0], gvs[1], lambda gene_id, variants: - hl.rbind( - variants.filter(lambda v: v.is_primary), - variants.filter(lambda v: v.is_secondary), - lambda v1s, v2s: - v1s.flatmap(lambda v1: - v2s \ - .filter(lambda v2: ~v2.is_primary | ~v1.is_secondary | (v1.key_ < v2.key_)) \ - .map(lambda v2: hl.tuple([gene_id, v1, v2])) - ) - ) - ) - ) - - """ After grouping by gene and pairing/filtering have array of tuples (gene_id, v1, v2) - (A, 1, 2) - (A, 1, 3) - (A, 2, 3) - (B, 2, 3) - """ - - variants = variants.group_by(lambda v: hl.tuple([v[1].key_, v[2].key_])) - variants = variants.values().map(lambda v: hl.rbind( - hl.set(v.map(lambda v: v[0])), - lambda comp_het_gene_ids: hl.struct( - v1=v[0][1].annotate(comp_het_gene_ids=comp_het_gene_ids), - v2=v[0][2].annotate(comp_het_gene_ids=comp_het_gene_ids), - ) - )) - - """ After grouping by pair have array of structs (v1, v2) annotated with comp_het_gene_ids - v1 | v2 | v<1/2>.comp_het_gene_ids - 1 | 2 | {A} - 1 | 3 | {A} - 2 | 3 | {A, B} - """ - - variants = self._filter_comp_het_families(variants) - - return hl.Table.parallelize( - variants.map(lambda v: hl.struct(**{GROUPED_VARIANTS_FIELD: hl.array([v.v1, v.v2])})) - ) - - def _filter_comp_het_families(self, variants, set_secondary_annotations=True): - variants = self._annotate_comp_het_valid_family_indices(variants) - return variants.filter( - lambda v: v.valid_family_indices.any(hl.is_defined) - ).map(lambda v: v.select( - v1=self._annotated_comp_het_variant(v.v1, v.valid_family_indices), - v2=self._annotated_comp_het_variant(v.v2, v.valid_family_indices, is_secondary=set_secondary_annotations), - )) - - def _annotate_comp_het_valid_family_indices(self, variants): - return variants.map(lambda v: v.annotate( - valid_family_indices=hl.enumerate(v.v1.family_entries).map(lambda x: x[0]).filter( - lambda i: self._is_valid_comp_het_family(v.v1, v.v2, i) - ) - )) - - def _annotated_comp_het_variant(self, variant, valid_family_indices, is_secondary=False): - if is_secondary and self._has_secondary_annotations and ALLOWED_TRANSCRIPTS in variant and ALLOWED_SECONDARY_TRANSCRIPTS in variant: - variant = variant.annotate(**{ALLOWED_TRANSCRIPTS: variant[ALLOWED_SECONDARY_TRANSCRIPTS]}) - - return variant.annotate( - family_entries=valid_family_indices.map(lambda i: variant.family_entries[i]), - ) - - @classmethod - def _gene_ids_expr(cls, ht, filtered_genes_only=False): - field = FILTERED_GENE_TRANSCRIPTS if filtered_genes_only and hasattr(ht, FILTERED_GENE_TRANSCRIPTS) else cls.TRANSCRIPTS_FIELD - return hl.set(ht[field].map(lambda t: t.gene_id)) - - def _is_valid_comp_het_family(self, v1, v2, family_index): - entries_1 = v1.family_entries[family_index] - entries_2 = v2.family_entries[family_index] - family_filters = [hl.is_defined(entries_1), hl.is_defined(entries_2)] - if self.max_unaffected_samples > 0: - family_filters.append(hl.enumerate(entries_1).all(lambda x: hl.any([ - (x[1].affected_id != UNAFFECTED_ID), *self._comp_het_entry_has_ref(x[1].GT, entries_2[x[0]].GT), - ]))) - if self._override_comp_het_alt: - family_filters.append(entries_1.extend(entries_2).all(lambda x: ~self.GENOTYPE_QUERY_MAP[ALT_ALT](x.GT))) - return hl.all(family_filters) - - def _comp_het_entry_has_ref(self, gt1, gt2): - return [self.GENOTYPE_QUERY_MAP[REF_REF](gt1), self.GENOTYPE_QUERY_MAP[REF_REF](gt2)] - - def _format_comp_het_results(self, ch_ht, annotation_fields): - formatted_grouped_variants = hl.array([ - self._format_results(ch_ht[GROUPED_VARIANTS_FIELD][0], annotation_fields=annotation_fields), - self._format_results(ch_ht[GROUPED_VARIANTS_FIELD][1], annotation_fields=annotation_fields), - ]) - ch_ht = ch_ht.annotate(**{GROUPED_VARIANTS_FIELD: hl.sorted(formatted_grouped_variants, key=lambda x: x._sort)}) - return ch_ht.annotate(_sort=ch_ht[GROUPED_VARIANTS_FIELD][0]._sort) - - def _format_results(self, ht, annotation_fields=None, **kwargs): - if annotation_fields is None: - annotation_fields = self.annotation_fields() - annotations = {k: v(ht) for k, v in annotation_fields.items()} - annotations.update({ - '_sort': self._sort_order(ht), - 'genomeVersion': self.GENOME_VERSION.replace('GRCh', ''), - }) - results = ht.annotate(**annotations) - return results.select(*self.CORE_FIELDS, *list(annotations.keys())) - - def format_search_ht(self): - ch_ht = None - annotation_fields = self.annotation_fields() - if self._comp_het_ht: - ch_ht = self._format_comp_het_results(self._comp_het_ht, annotation_fields) - - if self._ht: - ht = self._format_results(self._ht.key_by(), annotation_fields=annotation_fields) - if ch_ht: - ht = ht.union(ch_ht, unify=True) - else: - ht = ch_ht - return ht - - def search(self): - ht = self.format_search_ht() - - (total_results, collected) = (0, None) if ht is None else ht.aggregate((hl.agg.count(), hl.agg.take(ht.row, self._num_results, ordering=ht._sort))) - logger.info(f'Total hits: {total_results}. Fetched: {self._num_results}') - - return [] if collected is None else self._format_collected_rows(collected), total_results - - def _format_collected_rows(self, collected): - if self._has_comp_het_search: - return [row.get(GROUPED_VARIANTS_FIELD) or row.drop(GROUPED_VARIANTS_FIELD) for row in collected] - return collected - - def _sort_order(self, ht): - sort_expressions = self._get_sort_expressions(ht, XPOS) - if self._sort != XPOS: - sort_expressions = self._get_sort_expressions(ht, self._sort) + sort_expressions - return sort_expressions - - @staticmethod - def _format_prediction_sort_value(value): - return hl.or_else(-hl.float64(value), 0) - - def _get_sort_expressions(self, ht, sort): - if sort in self.SORTS: - return self.SORTS[sort](ht) - - if sort in self.PREDICTION_FIELDS_CONFIG: - prediction_path = self.PREDICTION_FIELDS_CONFIG[sort] - return [self._format_prediction_sort_value(ht[prediction_path.source][prediction_path.field])] - - if sort == OMIM_SORT: - return self._omim_sort(ht, hl.set(set(self._sort_metadata))) - - if self._sort_metadata: - return self._gene_rank_sort(ht, hl.dict(self._sort_metadata)) - - sort_field, sort_subfield = next( - ((field, config.get('sort_subfield', 'af')) for field, config in self.POPULATIONS.items() if config.get('sort') == sort), - (None, None), - ) - if sort_field: - return [hl.float64(self.population_expression(ht, sort_field)[sort_subfield])] - - return [] - - @classmethod - def _omim_sort(cls, r, omim_gene_set): - return [-cls._gene_ids_expr(r).intersection(omim_gene_set).size()] - - @classmethod - def _gene_rank_sort(cls, r, gene_ranks): - return [hl.min(cls._gene_ids_expr(r).map(gene_ranks.get))] - - @classmethod - def _gene_count_selects(cls): - return { - 'gene_ids': cls._gene_ids_expr, - 'families': cls.BASE_ANNOTATION_FIELDS[FAMILY_GUID_FIELD], - } - - def format_gene_count_hts(self): - hts = [] - selects = self._gene_count_selects() - if self._comp_het_ht: - ch_ht = self._comp_het_ht.explode(self._comp_het_ht[GROUPED_VARIANTS_FIELD]) - hts.append(ch_ht.select(**{k: v(ch_ht[GROUPED_VARIANTS_FIELD]) for k, v in selects.items()})) - if self._ht: - hts.append(self._ht.select(**{k: v(self._ht) for k, v in selects.items()})) - return hts - - def gene_counts(self): - hts = self.format_gene_count_hts() - ht = hts[0].key_by() - for sub_ht in hts[1:]: - ht = ht.union(sub_ht.key_by(), unify=True) - - ht = ht.explode('gene_ids').explode('families') - return ht.aggregate(hl.agg.group_by( - ht.gene_ids, hl.struct(total=hl.agg.count(), families=hl.agg.counter(ht.families)) - )) - - def _filter_variant_ids(self, ht, variant_ids): - return ht - - def lookup_variants(self, variant_ids, additional_annotations=None): - self._parse_intervals(intervals=None, variant_ids=variant_ids, variant_keys=variant_ids) - ht = self._read_table('annotations.ht', drop_globals=['versions']) - ht = self._filter_variant_ids(ht, variant_ids) - ht = ht.filter(hl.is_defined(ht[XPOS])) - - annotation_fields = { - k: v for k, v in self.annotation_fields(include_genotype_overrides=False).items() - if k not in {FAMILY_GUID_FIELD, GENOTYPES_FIELD} - } - if additional_annotations: - annotation_fields.update(additional_annotations) - formatted = self._format_results(ht.key_by(), annotation_fields=annotation_fields, include_genotype_overrides=False) - - return formatted.aggregate(hl.agg.take(formatted.row, len(variant_ids))) - - def _import_variant_projects_ht(self, variant_id, project_samples=None, **kwargs): - projects_ht, _ = self._import_and_filter_multiple_project_hts(project_samples, n_partitions=1) - return self._filter_variant_ids(projects_ht, [variant_id]).key_by() - - def _get_variant_project_data(self, variant_id, **kwargs): - projects_ht = self._import_variant_projects_ht(variant_id, **kwargs) - project_data = projects_ht.aggregate(hl.agg.take(projects_ht.row, 1)) - return project_data[0] if project_data else {} - - def lookup_variant(self, variant_id, **kwargs): - variants = self.lookup_variants([variant_id], additional_annotations=self._lookup_variant_annotations()) - if not variants: - raise HTTPNotFound() - variant = dict(variants[0]) - variant.update(self._get_variant_project_data(variant_id, variant=variant, **kwargs)) - return variant - - @staticmethod - def _lookup_variant_annotations(): - return {} diff --git a/hail_search/queries/gcnv.py b/hail_search/queries/gcnv.py deleted file mode 100644 index beae119b4b..0000000000 --- a/hail_search/queries/gcnv.py +++ /dev/null @@ -1,134 +0,0 @@ -import hail as hl - -from hail_search.constants import COMP_HET_ALT, HAS_ALT, HAS_REF, REF_REF, FILTERED_GENE_TRANSCRIPTS, \ - ALLOWED_TRANSCRIPTS, ALLOWED_SECONDARY_TRANSCRIPTS -from hail_search.queries.sv import SvHailTableQuery - - -class GcnvHailTableQuery(SvHailTableQuery): - - DATA_TYPE = 'SV_WES' - SV_TYPE_PREFIX = 'gCNV_' - - # gCNV data has no ref/ref calls so a missing entry indicates ref/ref - GENOTYPE_QUERY_MAP = { - **SvHailTableQuery.GENOTYPE_QUERY_MAP, - REF_REF: hl.is_missing, - HAS_REF: lambda gt: hl.is_missing(gt) | gt.is_het_ref(), - HAS_ALT: hl.is_defined, - COMP_HET_ALT: hl.is_defined, - } - MISSING_NUM_ALT = 0 - - GENOTYPE_FIELDS = { - **SvHailTableQuery.GENOTYPE_FIELDS, - **{f.lower(): f for f in ['QS', 'defragged']}, - } - del GENOTYPE_FIELDS['gq'] - GENOTYPE_QUERY_FIELDS = {} - GENOTYPE_OVERRIDE_FIELDS = { - 'start': (hl.min, lambda r: r.start_locus.position), - 'end': (hl.max, lambda r: r.end_locus.position), - 'num_exon': (hl.max, lambda r: r.num_exon), - 'gene_ids': ( - lambda entry_gene_ids: entry_gene_ids.fold(lambda s1, s2: s1.union(s2), hl.empty_set(hl.tstr)), - lambda r: hl.missing(hl.tset(hl.tstr)), - ), - } - COMPUTED_GENOTYPE_FIELDS = { - **SvHailTableQuery.COMPUTED_GENOTYPE_FIELDS, - **{k: lambda entry, field, r: hl.or_missing( - hl.is_missing(r[field]) | (r[field] != entry[f'sample_{field}']), entry[f'sample_{field}'] - ) for k in GENOTYPE_OVERRIDE_FIELDS.keys()}, - } - COMPUTED_GENOTYPE_FIELDS['prev_overlap'] = COMPUTED_GENOTYPE_FIELDS.pop('prev_num_alt') - - CORE_FIELDS = SvHailTableQuery.CORE_FIELDS[:-1] - BASE_ANNOTATION_FIELDS = { - **SvHailTableQuery.BASE_ANNOTATION_FIELDS, - 'pos': lambda r: r.start, - 'end': lambda r: r.end, - 'numExon': lambda r: r.num_exon, - } - del BASE_ANNOTATION_FIELDS['bothsidesSupport'] - - TRANSCRIPTS_ENUM_FIELD = SvHailTableQuery.ENUM_ANNOTATION_FIELDS[SvHailTableQuery.TRANSCRIPTS_FIELD] - ENUM_ANNOTATION_FIELDS = {SvHailTableQuery.TRANSCRIPTS_FIELD: { - **TRANSCRIPTS_ENUM_FIELD, - 'format_array_values': lambda values, r: GcnvHailTableQuery.TRANSCRIPTS_ENUM_FIELD['format_array_values']( - hl.if_else(hl.is_missing(r.gene_ids), values, values.filter(lambda t: r.gene_ids.contains(t.geneId))), r, - ), - }} - - POPULATIONS = {k: v for k, v in SvHailTableQuery.POPULATIONS.items() if k != 'gnomad_svs'} - - @classmethod - def _get_genotype_override_field(cls, r, field): - agg, get_default = cls.GENOTYPE_OVERRIDE_FIELDS[field] - sample_field = f'sample_{field}' - entries = r.family_entries.flatmap(lambda x: x) - return hl.if_else( - entries.any(lambda g: hl.is_defined(g.GT) & hl.is_missing(g[sample_field])), - get_default(r), agg(entries.map(lambda g: g[sample_field])) - ) - - def _format_results(self, ht, *args, include_genotype_overrides=True, **kwargs): - ht = ht.annotate(**{ - k: self._get_genotype_override_field(ht, k) if include_genotype_overrides - else self.GENOTYPE_OVERRIDE_FIELDS[k][1](ht) - for k in self.GENOTYPE_OVERRIDE_FIELDS - }) - return super()._format_results(ht, *args, **kwargs) - - def get_allowed_sv_type_ids(self, sv_types): - return super().get_allowed_sv_type_ids([ - type.replace(self.SV_TYPE_PREFIX, '') for type in sv_types if type.startswith(self.SV_TYPE_PREFIX) - ]) - - def _annotate_families_table_annotations(self, families_ht, annotations_ht, is_comp_het=False): - ht = super()._annotate_families_table_annotations(families_ht, annotations_ht) - transcript_fields = [ - transcript_field for transcript_field in [ - ALLOWED_TRANSCRIPTS, ALLOWED_SECONDARY_TRANSCRIPTS, FILTERED_GENE_TRANSCRIPTS, - ] if hasattr(ht, transcript_field) - ] - if not transcript_fields: - return ht - - ht = ht.annotate(all_gene_ids=self._get_genotype_override_field(ht, 'gene_ids')) - for transcript_field in transcript_fields: - transcript_has_gene = lambda t: hl.is_missing(ht.all_gene_ids) | ht.all_gene_ids.contains(t.gene_id) - if transcript_field == FILTERED_GENE_TRANSCRIPTS: - ht = ht.filter(ht[transcript_field].any(transcript_has_gene)) - else: - ht = ht.annotate(**{transcript_field: ht[transcript_field].filter(transcript_has_gene)}) - if not is_comp_het: - # Will trigger re-filtering the ht for allowed annotations - self._has_secondary_annotations = True - - return ht.drop('all_gene_ids') - - def _filter_by_gene_ids(self, ht, gene_ids): - if not hasattr(ht, 'family_entries'): - return super()._filter_by_gene_ids(ht, gene_ids) - - # Gene and annotation filters should only consider genes present in the genotype override gene_ids - row_gene_ids = self._gene_ids_expr(ht) - - if gene_ids is None: - return self._annotate_filtered_gene_transcripts(ht, row_gene_ids) - - return super()._filter_by_gene_ids(ht, gene_ids.intersection(row_gene_ids)) - - @classmethod - def _gene_ids_expr(cls, ht, filtered_genes_only=False): - gene_ids_expr = super()._gene_ids_expr(ht) - if filtered_genes_only: - return gene_ids_expr - return hl.or_else( - cls._get_genotype_override_field(ht, 'gene_ids'), - gene_ids_expr, - ) - - def _additional_annotation_fields(self): - return {} diff --git a/hail_search/queries/mito.py b/hail_search/queries/mito.py deleted file mode 100644 index 59efeeaaef..0000000000 --- a/hail_search/queries/mito.py +++ /dev/null @@ -1,558 +0,0 @@ -from collections import defaultdict - -from aiohttp.web import HTTPNotFound -import hail as hl -import logging -import os - -from hail_search.constants import ABSENT_PATH_SORT_OFFSET, CLINVAR_KEY, CLINVAR_LIKELY_PATH_FILTER, \ - CLINVAR_PATH_FILTER, \ - CLINVAR_PATH_RANGES, CLINVAR_PATH_SIGNIFICANCES, ALLOWED_TRANSCRIPTS, ALLOWED_SECONDARY_TRANSCRIPTS, \ - PATHOGENICTY_SORT_KEY, CONSEQUENCE_SORT, FILTERED_GENE_TRANSCRIPTS, \ - PATHOGENICTY_HGMD_SORT_KEY, MAX_LOAD_INTERVALS -from hail_search.definitions import SampleType -from hail_search.queries.base import BaseHailTableQuery, PredictionPath, QualityFilterFormat, MAX_PARTITIONS - -REFERENCE_DATASETS_DIR = os.environ.get('REFERENCE_DATASETS_DIR', '/seqr/seqr-reference-data') - -logger = logging.getLogger(__name__) - - -def _clinvar_sort(clinvar_field, r): - return hl.or_else(r[clinvar_field].pathogenicity_id, ABSENT_PATH_SORT_OFFSET) - - -class MitoHailTableQuery(BaseHailTableQuery): - - DATA_TYPE = 'MITO' - KEY_FIELD = ('locus', 'alleles') - - TRANSCRIPTS_FIELD = 'sorted_transcript_consequences' - TRANSCRIPT_CONSEQUENCE_FIELD = 'consequence_term' - GENOTYPE_FIELDS = { - 'dp': 'DP', - 'hl': 'HL', - 'mitoCn': 'mito_cn', - 'contamination': 'contamination', - } - QUALITY_FILTER_FORMAT = { - 'HL': QualityFilterFormat(scale=100), - } - - POPULATION_FIELDS = {'helix': 'helix_mito', 'seqr': 'gt_stats'} - POPULATIONS = {} - for pop, sort in {'seqr': 'callset_af', 'gnomad_mito': 'gnomad', 'helix': None}.items(): - pop_het = f'{pop}_heteroplasmy' - POPULATIONS.update({ - pop: {'af': 'AF_hom', 'ac': 'AC_hom', 'hom': None, 'hemi': None, 'het': None, 'sort': sort}, - pop_het: { - 'af': 'AF_het', 'ac': 'AC_het', 'max_hl': None if pop == 'seqr' else 'max_hl', - 'hom': None, 'hemi': None, 'het': None, - }, - }) - POPULATION_FIELDS[pop_het] = POPULATION_FIELDS.get(pop, pop) - POPULATIONS['seqr']['sort_subfield'] = 'ac' - PREDICTION_FIELDS_CONFIG = { - 'apogee': PredictionPath('mitimpact', 'score'), - 'haplogroup_defining': PredictionPath('haplogroup', 'is_defining', lambda v: hl.or_missing(v, 'Y')), - 'hmtvar': PredictionPath('hmtvar', 'score'), - 'mitotip': PredictionPath('mitotip', 'trna_prediction'), - 'mut_taster': PredictionPath('dbnsfp', 'MutationTaster_pred'), - 'sift': PredictionPath('dbnsfp', 'SIFT_score'), - 'mlc': PredictionPath('local_constraint_mito', 'score'), - } - - PATHOGENICITY_FILTERS = { - CLINVAR_KEY: ('pathogenicity', CLINVAR_PATH_RANGES), - } - - GLOBALS = BaseHailTableQuery.GLOBALS + ['versions'] - CORE_FIELDS = BaseHailTableQuery.CORE_FIELDS + ['rsid'] - MITO_ANNOTATION_FIELDS = { - 'commonLowHeteroplasmy': lambda r: r.common_low_heteroplasmy, - 'mitomapPathogenic': lambda r: r.mitomap.pathogenic, - } - BASE_ANNOTATION_FIELDS = { - 'chrom': lambda r: r.locus.contig.replace("^chr", ""), - 'pos': lambda r: r.locus.position, - 'ref': lambda r: r.alleles[0], - 'alt': lambda r: r.alleles[1], - 'mainTranscriptId': lambda r: r.sorted_transcript_consequences.first().transcript_id, - 'selectedMainTranscriptId': lambda r: hl.or_missing( - r.selected_transcript != r.sorted_transcript_consequences.first(), r.selected_transcript.transcript_id, - ), - **MITO_ANNOTATION_FIELDS, - **BaseHailTableQuery.BASE_ANNOTATION_FIELDS, - } - ENUM_ANNOTATION_FIELDS = { - CLINVAR_KEY: { - 'response_key': CLINVAR_KEY, - 'include_version': True, - 'annotate_value': lambda value, enum: { - 'conflictingPathogenicities': MitoHailTableQuery._format_enum( - value, 'conflictingPathogenicities', enum, enum_keys=['pathogenicity']), - }, - }, - TRANSCRIPTS_FIELD: { - **BaseHailTableQuery.ENUM_ANNOTATION_FIELDS['transcripts'], - 'annotate_value': lambda transcript, *args: {'major_consequence': transcript.consequence_terms.first()}, - 'format_array_values': lambda values, *args: BaseHailTableQuery.ENUM_ANNOTATION_FIELDS['transcripts']['format_array_values'](values).map_values( - lambda transcripts: hl.enumerate(transcripts).starmap(lambda i, t: t.annotate(transcriptRank=i)) - ), - } - } - - CLINVAR_SORT = _clinvar_sort - SORTS = { - CONSEQUENCE_SORT: lambda r: [ - hl.min(r.sorted_transcript_consequences.flatmap(lambda t: t.consequence_term_ids)), - hl.min(r.selected_transcript.consequence_term_ids), - ], - PATHOGENICTY_SORT_KEY: lambda r: [_clinvar_sort(CLINVAR_KEY, r)], - **BaseHailTableQuery.SORTS, - } - SORTS[PATHOGENICTY_HGMD_SORT_KEY] = SORTS[PATHOGENICTY_SORT_KEY] - - PREFILTER_TABLES = { - CLINVAR_KEY: 'clinvar_path_variants.ht', - } - - def _import_and_filter_entries_ht( - self, project_guid: str, num_families: int, project_sample_type_data, **kwargs - ) -> tuple[hl.Table, hl.Table]: - sample_types = set(project_sample_type_data.keys()) - if len(sample_types) == 1: - # Import and filter entry table for one family or one project with one sample type - return super()._import_and_filter_entries_ht(project_guid, num_families, project_sample_type_data, **kwargs) - - self._has_both_sample_types = True - entries = {} - for sample_type in sample_types: - entries[sample_type] = self._load_family_or_project_ht( - num_families, project_guid, project_sample_type_data, sample_type, **kwargs - ) - - return self._filter_entries_ht_both_sample_types( - *entries[SampleType.WES.value], *entries[SampleType.WGS.value], **kwargs - ) - - def _import_and_filter_multiple_project_hts( - self, project_samples: dict, n_partitions=MAX_PARTITIONS, **kwargs - ) -> tuple[hl.Table, hl.Table]: - single_sample_type_project_samples = {} - both_sample_type_project_samples = defaultdict(dict) - for project_guid, sample_dict in project_samples.items(): - if len(sample_dict) == 1: - single_sample_type_project_samples[project_guid] = sample_dict - else: - both_sample_type_project_samples[SampleType.WES][project_guid] = {SampleType.WES: sample_dict[SampleType.WES]} - both_sample_type_project_samples[SampleType.WGS][project_guid] = {SampleType.WGS: sample_dict[SampleType.WGS]} - - filtered_project_hts = [] - filtered_comp_het_project_hts = [] - - # Process projects with only one sample type separately - if single_sample_type_project_samples: - ht, ch_ht = super()._import_and_filter_multiple_project_hts(single_sample_type_project_samples, n_partitions, **kwargs) - if ht is not None: - filtered_project_hts.append(ht) - if ch_ht is not None: - filtered_comp_het_project_hts.append(ch_ht) - - if both_sample_type_project_samples: - self._has_both_sample_types = True - wes_entries = self._load_project_hts( - both_sample_type_project_samples[SampleType.WES], n_partitions, **kwargs, - ) - wgs_entries = self._load_project_hts( - both_sample_type_project_samples[SampleType.WGS], n_partitions, **kwargs, - ) - for i, (wes_ht, wes_project_samples) in enumerate(wes_entries): - wgs_ht, wgs_project_samples = wgs_entries[i] - ht, comp_het_ht = self._filter_entries_ht_both_sample_types( - wes_ht, wes_project_samples, wgs_ht, wgs_project_samples, - is_merged_ht=True, **kwargs - ) - if ht is not None: - filtered_project_hts.append(ht) - if comp_het_ht is not None: - filtered_comp_het_project_hts.append(comp_het_ht) - - return self._merge_filtered_hts(filtered_comp_het_project_hts, filtered_project_hts, n_partitions) - - def _filter_entries_ht_both_sample_types( - self, wes_ht, wes_project_samples, wgs_ht, wgs_project_samples, inheritance_filter=None, quality_filter=None, - is_merged_ht=False, **kwargs - ): - wes_ht, sorted_wes_family_sample_data = self._add_entry_sample_families(wes_ht, wes_project_samples, is_merged_ht) - wgs_ht, sorted_wgs_family_sample_data = self._add_entry_sample_families(wgs_ht, wgs_project_samples, is_merged_ht) - wes_ht = wes_ht.rename({'family_entries': SampleType.WES.family_entries_field}) - wgs_ht = wgs_ht.rename({'family_entries': SampleType.WGS.family_entries_field}) - ht = wes_ht.join(wgs_ht, how='outer') - - sample_types = [ - (SampleType.WES, sorted_wes_family_sample_data), - (SampleType.WGS, sorted_wgs_family_sample_data) - ] - for sample_type, _ in sample_types: - ht = self._filter_quality( - ht, quality_filter, - annotation=sample_type.passes_quality_field, entries_ht_field=sample_type.family_entries_field, **kwargs - ) - - ch_ht = None - family_idx_map = defaultdict(dict) - for sample_type, sorted_family_sample_data in sample_types: - ht = self._annotate_initial_passes_inheritance(ht, sample_type) - ch_ht = self._annotate_initial_passes_inheritance(ch_ht, sample_type) - ht, ch_ht = self._filter_inheritance( - ht, ch_ht, inheritance_filter, sorted_family_sample_data, - annotation=sample_type.passes_inheritance_field, entries_ht_field=sample_type.family_entries_field, - family_passes_inheritance_filter=self._get_family_passes_inheritance_filter_both_sample_types - ) - for family_idx, samples in enumerate(sorted_family_sample_data): - family_guid = samples[0]['familyGuid'] - family_idx_map[family_guid][sample_type.value] = family_idx - - family_idx_map = hl.dict(family_idx_map) - ht = self._apply_multi_sample_type_entry_filters(ht, family_idx_map) - ch_ht = self._apply_multi_sample_type_entry_filters(ch_ht, family_idx_map) - return ht, ch_ht - - @staticmethod - def _annotate_initial_passes_inheritance(ht, sample_type): - if ht is None: - return ht - - return ht.annotate(**{ - sample_type.passes_inheritance_field: ht[sample_type.family_entries_field].map( - lambda family_entries: hl.array( - hl.range(0, hl.len(family_entries)).map(lambda _: True) - ) - )}) - - def _get_family_passes_inheritance_filter_both_sample_types( - self, entry_indices, family_idx, genotype, family_samples, ht, annotation - ): - return hl.enumerate(ht[annotation][family_idx]).starmap( - lambda sample_idx, passes: (hl.case() - .when(~entry_indices.get(family_idx).contains(sample_idx), passes) - .when(~self.GENOTYPE_QUERY_MAP[genotype](family_samples[sample_idx].GT), False) - .default(passes)) - ) - - def _apply_multi_sample_type_entry_filters(self, ht, family_idx_map): - if ht is None: - return ht - - for sample_type in SampleType: - ht = ht.annotate(**{ - sample_type.family_entries_field: hl.enumerate(ht[sample_type.family_entries_field]).starmap( - lambda family_idx, family_samples: hl.or_missing( - hl.bind(lambda other_sample_type_family_idx: (( - self._family_has_valid_quality(ht, sample_type, family_idx) | - self._family_has_valid_quality(ht, sample_type.other_sample_type, other_sample_type_family_idx) - ) & - self._family_has_valid_inheritance(ht, sample_type, family_idx, other_sample_type_family_idx) & - self._family_has_valid_inheritance(ht, sample_type.other_sample_type, other_sample_type_family_idx, family_idx) - ), family_idx_map.get(hl.coalesce(family_samples)[0]['familyGuid']).get(sample_type.other_sample_type.value), - ), family_samples) - )}) - - # Merge family entries and filters from both sample types - ht = ht.transmute( - family_entries=hl.coalesce( - ht[SampleType.WES.family_entries_field], hl.empty_array(ht[SampleType.WES.family_entries_field].dtype.element_type) - ).extend(hl.coalesce( - ht[SampleType.WGS.family_entries_field], hl.empty_array(ht[SampleType.WGS.family_entries_field].dtype.element_type) - )).filter(lambda entries: entries.any(hl.is_defined)) - ) - ht = ht.select('family_entries') - ht = ht.select_globals('family_guids') - - # Filter out families with no valid entries in either sample type - return ht.filter(ht.family_entries.any(hl.is_defined)) - - @staticmethod - def _family_has_valid_quality(ht, sample_type, sample_type_family_idx): - return ( - hl.is_defined(sample_type_family_idx) & - hl.is_defined(ht[sample_type.passes_quality_field][sample_type_family_idx]) - ) - - @staticmethod - def _family_has_valid_inheritance(ht, sample_type, family_idx, other_sample_type_family_idx): - return hl.bind( - lambda sample_type_fail_samples, other_sample_type_pass_samples: ( - sample_type_fail_samples.all(other_sample_type_pass_samples.contains) - ), hl.enumerate(ht[sample_type.family_entries_field][family_idx]).starmap( - lambda sample_idx, sample: hl.or_missing( - ~ht[sample_type.passes_inheritance_field][family_idx][sample_idx], - sample['sampleId'], - ) - ).filter(hl.is_defined), - hl.enumerate(ht[sample_type.other_sample_type.family_entries_field][other_sample_type_family_idx]).starmap( - lambda sample_idx, sample: hl.or_missing( - ht[sample_type.other_sample_type.passes_inheritance_field][other_sample_type_family_idx][sample_idx], - sample['sampleId'], - ) - ).filter(hl.is_defined), - ) - - def _annotate_comp_het_valid_family_indices(self, variants): - variants = super()._annotate_comp_het_valid_family_indices(variants) - if not self._has_both_sample_types: - return variants - - # Get valid comp het familyGuids - variants = variants.map(lambda v: v.annotate( - valid_family_guids=v.valid_family_indices.map( - lambda i: v.v1.family_entries[i][0]['familyGuid'] - ) - )) - # Re-annotate valid_family_indices based on valid familyGuids - return variants.map(lambda v: v.annotate( - valid_family_indices=hl.enumerate(v.v1.family_entries).filter( - lambda x: v.valid_family_guids.contains(x[1][0]['familyGuid']) - ).map(lambda x: x[0]) - )) - - - def _get_sample_genotype(self, samples, r=None, include_genotype_overrides=False, select_fields=None, **kwargs): - if not self._has_both_sample_types: - return super()._get_sample_genotype(samples, r, include_genotype_overrides, select_fields) - - return samples.map(lambda sample: self._select_genotype_for_sample( - sample, r, include_genotype_overrides, select_fields - )) - - @staticmethod - def _selected_main_transcript_expr(ht): - comp_het_gene_ids = getattr(ht, 'comp_het_gene_ids', None) - if comp_het_gene_ids is not None: - return hl.coalesce(*[ - ht[field].find(lambda t: comp_het_gene_ids.contains(t.gene_id)) for field in [ - ALLOWED_TRANSCRIPTS, ALLOWED_SECONDARY_TRANSCRIPTS, 'sorted_transcript_consequences', - ] if hasattr(ht, field) - ]) - - main_transcript = getattr(ht, FILTERED_GENE_TRANSCRIPTS, ht.sorted_transcript_consequences).first() - if hasattr(ht, ALLOWED_TRANSCRIPTS): - return hl.or_else(ht[ALLOWED_TRANSCRIPTS].first(), main_transcript) - return main_transcript - - def __init__(self, *args, **kwargs): - self._filter_hts = {} - self._has_both_sample_types = False - super().__init__(*args, **kwargs) - - def _get_family_passes_quality_filter(self, quality_filter, ht, pathogenicity=None, **kwargs): - passes_quality = super()._get_family_passes_quality_filter(quality_filter, ht) - clinvar_path_ht = False if passes_quality is None else self._get_loaded_clinvar_prefilter_ht(pathogenicity) - if not clinvar_path_ht: - return passes_quality - - return lambda entries: hl.is_defined(clinvar_path_ht[ht.key]) | passes_quality(entries) - - def _get_loaded_filter_ht(self, key, get_filters, **kwargs): - if self._filter_hts.get(key) is None: - ht_filter = get_filters(**kwargs) - if ht_filter is False: - self._filter_hts[key] = False - else: - ht = self._read_table(f'{self.PREFILTER_TABLES[key]}') - if ht_filter is not True: - ht = ht.filter(ht_filter(ht)) - self._filter_hts[key] = ht - - return self._filter_hts[key] - - @classmethod - def _get_table_dir(cls, path): - if any(prefilter_table_path in path for prefilter_table_path in cls.PREFILTER_TABLES.values()): - return REFERENCE_DATASETS_DIR - return super()._get_table_dir(path) - - def _get_loaded_clinvar_prefilter_ht(self, pathogenicity): - return self._get_loaded_filter_ht( - CLINVAR_KEY, self._get_clinvar_prefilter, pathogenicity=pathogenicity) - - def _get_clinvar_prefilter(self, pathogenicity=None): - clinvar_path_filters = self._get_clinvar_path_filters(pathogenicity) - if not clinvar_path_filters: - return False - - if CLINVAR_LIKELY_PATH_FILTER not in clinvar_path_filters: - return lambda ht: ht.is_pathogenic - elif CLINVAR_PATH_FILTER not in clinvar_path_filters: - return lambda ht: ht.is_likely_pathogenic - return True - - def _filter_variant_ids(self, ht, variant_ids): - if len(variant_ids) == 1: - variant_id_q = ht.alleles == [variant_ids[0][2], variant_ids[0][3]] - else: - variant_id_q = hl.any([ - (ht.locus == hl.locus(f'chr{chrom}' if self._should_add_chr_prefix() else chrom, pos, reference_genome=self.GENOME_VERSION)) & - (ht.alleles == [ref, alt]) - for chrom, pos, ref, alt in variant_ids - ]) - return ht.filter(variant_id_q) - - def _parse_variant_keys(self, variant_keys): - return None - - def _prefilter_entries_table(self, ht, parsed_intervals=None, exclude_intervals=False, variant_ids=None, **kwargs): - num_intervals = len(parsed_intervals or []) - if exclude_intervals and parsed_intervals: - ht = hl.filter_intervals(ht, parsed_intervals, keep=False) - elif num_intervals >= MAX_LOAD_INTERVALS: - ht = hl.filter_intervals(ht, parsed_intervals) - - if variant_ids: - ht = self._filter_variant_ids(ht, variant_ids) - - if '_n_partitions' not in self._load_table_kwargs and num_intervals > self._n_partitions: - ht = ht.naive_coalesce(self._n_partitions) - - return ht - - def _get_allowed_consequence_ids(self, annotations): - consequence_ids = super()._get_allowed_consequence_ids(annotations) - canonical_consequences = { - c.replace('__canonical', '') for consequences in annotations.values() - if consequences for c in consequences if c.endswith('__canonical') - } - if canonical_consequences: - canonical_consequence_ids = super()._get_allowed_consequence_ids({'canonical': canonical_consequences}) - canonical_consequence_ids -= consequence_ids - consequence_ids.update({f'{cid}__canonical' for cid in canonical_consequence_ids}) - return consequence_ids - - @staticmethod - def _get_allowed_transcripts_filter(allowed_consequence_ids): - canonical_consequences = set() - any_consequences = set() - for c in allowed_consequence_ids: - if str(c).endswith('__canonical'): - canonical_consequences.add(int(c.replace('__canonical', ''))) - else: - any_consequences.add(c) - - all_consequence_ids = None - if canonical_consequences: - all_consequence_ids = hl.set({*canonical_consequences, *any_consequences}) - - allowed_consequence_ids = hl.set(any_consequences) if any_consequences else hl.empty_set(hl.tint) - return lambda tc: tc.consequence_term_ids.any( - (hl.if_else(hl.is_defined(tc.canonical), all_consequence_ids, allowed_consequence_ids) - if canonical_consequences else allowed_consequence_ids - ).contains) - - def _filter_excluded(self, ht, exclude): - for key in self.PATHOGENICITY_FILTERS.keys(): - path_terms = (exclude or {}).get(key) - if path_terms: - ht = ht.filter(hl.is_missing(ht[key]) | ~self._has_path_expr(ht, path_terms, key)) - return ht - - def _get_annotation_override_fields(self, annotations, pathogenicity=None, **kwargs): - annotation_overrides = super()._get_annotation_override_fields(annotations, **kwargs) - for key in self.PATHOGENICITY_FILTERS.keys(): - path_terms = (pathogenicity or {}).get(key) - if path_terms: - annotation_overrides[key] = path_terms - return annotation_overrides - - def _get_annotation_override_filters(self, ht, annotation_overrides): - annotation_filters = [] - - for key in self.PATHOGENICITY_FILTERS.keys(): - if key in annotation_overrides: - annotation_filters.append(self._has_path_expr(ht, annotation_overrides[key], key)) - - return annotation_filters - - def _frequency_override_filter(self, ht, pathogenicity): - path_terms = self._get_clinvar_path_filters(pathogenicity) - return self._has_path_expr(ht, path_terms, CLINVAR_KEY) if path_terms else None - - @staticmethod - def _get_clinvar_path_filters(pathogenicity): - return { - f for f in (pathogenicity or {}).get(CLINVAR_KEY) or [] if f in CLINVAR_PATH_SIGNIFICANCES - } - - def _has_path_expr(self, ht, terms, field): - subfield, range_configs = self.PATHOGENICITY_FILTERS[field] - enum_lookup = self._get_enum_lookup(field, subfield) - - ranges = [[None, None]] - for path_filter, start, end in range_configs: - if path_filter in terms: - ranges[-1][1] = len(enum_lookup) if end is None else enum_lookup[end] - if ranges[-1][0] is None: - ranges[-1][0] = enum_lookup[start] - elif ranges[-1] != [None, None]: - ranges.append([None, None]) - - ranges = [r for r in ranges if r[0] is not None] - value = ht[field][f'{subfield}_id'] - return hl.any(lambda r: (value >= r[0]) & (value <= r[1]), ranges) - - def _format_results(self, ht, *args, **kwargs): - ht = ht.annotate(selected_transcript=self._selected_main_transcript_expr(ht)) - return super()._format_results(ht, *args, **kwargs) - - @classmethod - def _omim_sort(cls, r, omim_gene_set): - return [ - hl.if_else(omim_gene_set.contains(r.selected_transcript.gene_id), 0, 1), - ] + super()._omim_sort(r, omim_gene_set) - - @classmethod - def _gene_rank_sort(cls, r, gene_ranks): - return [gene_ranks.get(r.selected_transcript.gene_id)] + super()._gene_rank_sort(r, gene_ranks) - - def _import_variant_projects_ht(self, variant_id, *args, **kwargs): - # Get all the project-families for the looked up variant formatted as a dict of dicts: - # {: {: {: True}, : {: True}}, : ...} - lookup_ht = self._read_table('lookup.ht', skip_missing_field='project_stats') - lookup_ht = self._filter_variant_ids(lookup_ht, [variant_id]) - if lookup_ht is None: - raise HTTPNotFound() - variant_projects = lookup_ht.aggregate(hl.agg.take( - hl.dict(hl.enumerate(lookup_ht.project_stats).starmap(lambda i, ps: ( - lookup_ht.project_sample_types[i], - hl.enumerate(ps).starmap( - lambda j, s: hl.or_missing(self._stat_has_non_ref(s), j) - ).filter(hl.is_defined), - )).filter( - lambda x: x[1].any(hl.is_defined) - ).starmap(lambda project_key, family_indices: ( - project_key, - hl.dict(family_indices.map(lambda j: (lookup_ht.project_families[project_key][j], True))), - )).group_by( - lambda x: x[0][0] - ).map_values( - lambda project_data: hl.dict(project_data.starmap( - lambda project_key, families: (project_key[1], families) - )))), 1) - ) - - # Variant can be present in the lookup table with only ref calls, so is still not present in any projects - if not (variant_projects and variant_projects[0]): - raise HTTPNotFound() - variant_projects = variant_projects[0] - - self._has_both_sample_types = True - logger.info(f'Looking up {self.DATA_TYPE} variant in {len(variant_projects)} projects') - - projects_ht = super()._import_variant_projects_ht(variant_id, project_samples=variant_projects) - return projects_ht.select(familyGenotypes=hl.dict(projects_ht.family_entries.map( - lambda entries: (entries.first().familyGuid, self._get_sample_genotype(entries.filter(hl.is_defined))) - ))) - - @staticmethod - def _stat_has_non_ref(s): - return (s.heteroplasmic_samples > 0) | (s.homoplasmic_samples > 0) diff --git a/hail_search/queries/multi_data_types.py b/hail_search/queries/multi_data_types.py deleted file mode 100644 index 660bd70d0a..0000000000 --- a/hail_search/queries/multi_data_types.py +++ /dev/null @@ -1,216 +0,0 @@ -import hail as hl - -from hail_search.constants import ALT_ALT, REF_REF, CONSEQUENCE_SORT, OMIM_SORT, GROUPED_VARIANTS_FIELD, GENOME_VERSION_GRCh38 -from hail_search.queries.base import BaseHailTableQuery -from hail_search.queries.mito import MitoHailTableQuery -from hail_search.queries.snv_indel import SnvIndelHailTableQuery -from hail_search.queries.snv_indel_37 import SnvIndelHailTableQuery37 -from hail_search.queries.sv import SvHailTableQuery -from hail_search.queries.gcnv import GcnvHailTableQuery - -QUERY_CLASSES = [SnvIndelHailTableQuery, SnvIndelHailTableQuery37, MitoHailTableQuery, SvHailTableQuery, GcnvHailTableQuery] -QUERY_CLASS_MAP = {(cls.DATA_TYPE, cls.GENOME_VERSION): cls for cls in QUERY_CLASSES} -SNV_INDEL_DATA_TYPE = SnvIndelHailTableQuery.DATA_TYPE - - -class MultiDataTypeHailTableQuery(BaseHailTableQuery): - - LOADED_GLOBALS = True - - def __init__(self, sample_data, *args, **kwargs): - self._data_type_queries = {} - self._merged_comp_het_ht = None - self._sv_type_del_id = None - self._sv_data_type = next( - (dt for dt in [SvHailTableQuery.DATA_TYPE, GcnvHailTableQuery.DATA_TYPE] if dt in sample_data), None, - ) - super().__init__(sample_data, *args, **kwargs) - - def _load_filtered_table(self, sample_data, *args, **kwargs): - has_merged_comp_het = self._has_comp_het_search and SNV_INDEL_DATA_TYPE in sample_data and self._sv_data_type - overlapped_families = {s['family_guid'] for s in sample_data[SNV_INDEL_DATA_TYPE]}.intersection( - {s['family_guid'] for s in sample_data[self._sv_data_type]} - ) if has_merged_comp_het else None - - data_type_families = {} - # Multi project SV search skips projects with no matched variants, so run SV search first to reduce search space - for data_type in [self._sv_data_type, *sample_data.keys()]: - if data_type is None or data_type in self._data_type_queries: - continue - self._data_type_queries[data_type] = QUERY_CLASS_MAP[(data_type, GENOME_VERSION_GRCh38)]( - sample_data[data_type], *args, override_comp_het_alt=data_type == SNV_INDEL_DATA_TYPE, - overlapped_families=overlapped_families, sort=self._sort, sort_metadata=self._sort_metadata, - inheritance_mode=self._inheritance_mode, num_results=self._num_results, **kwargs - ) - if has_merged_comp_het and data_type in {self._sv_data_type, SNV_INDEL_DATA_TYPE}: - ht = self._data_type_queries[data_type].unfiltered_comp_het_ht - if ht is None: - has_merged_comp_het = False - overlapped_families = set() - else: - data_type_families[data_type] = hl.eval(ht.family_guids) - overlapped_families = overlapped_families.intersection(data_type_families[data_type]) - - if not (has_merged_comp_het and overlapped_families): - return - - variant_query = self._data_type_queries[SNV_INDEL_DATA_TYPE] - sv_query = self._data_type_queries[self._sv_data_type] - self.max_unaffected_samples = min(variant_query.max_unaffected_samples, sv_query.max_unaffected_samples) - merged_ht = self._filter_data_type_comp_hets( - variant_query, sv_query, sorted(overlapped_families), data_type_families, - ) - if merged_ht is not None: - self._merged_comp_het_ht = merged_ht.key_by() - - def _filter_data_type_comp_hets(self, variant_query, sv_query, overlapped_families, data_type_families): - variant_ht = self._family_filtered_ch_ht( - variant_query.unfiltered_comp_het_ht, overlapped_families, data_type_families[SNV_INDEL_DATA_TYPE], - ) - sv_ht = self._family_filtered_ch_ht( - sv_query.unfiltered_comp_het_ht, overlapped_families, data_type_families[self._sv_data_type], - ) - - sv_type_del_ids = sv_query.get_allowed_sv_type_ids([f'{getattr(sv_query, "SV_TYPE_PREFIX", "")}DEL']) - self._sv_type_del_id = list(sv_type_del_ids)[0] if sv_type_del_ids else None - - variant_samples_by_family = variant_query.entry_samples_by_family_guid - sv_samples_by_family = sv_query.entry_samples_by_family_guid - if any(f for f in overlapped_families if variant_samples_by_family[f] != sv_samples_by_family[f]): - sv_sample_indices = hl.array([[ - sv_samples_by_family[f].index(s) if s in sv_samples_by_family[f] else None - for s in variant_samples_by_family[f] - ] for f in overlapped_families]) - sv_ht = sv_ht.annotate(family_entries=hl.enumerate(sv_sample_indices).starmap( - lambda family_i, indices: hl.bind( - lambda family_entry: hl.or_missing( - hl.is_defined(family_entry), - indices.map(lambda sample_i: family_entry[sample_i]), - ), - sv_ht.family_entries[family_i], - ) - )) - - variant_ch_ht = variant_ht.group_by('gene_ids').aggregate(v1=hl.agg.collect(variant_ht.row)) - sv_ch_ht = sv_ht.group_by('gene_ids').aggregate(v2=hl.agg.collect(sv_ht.row)) - - variants = variant_ch_ht.join(sv_ch_ht).collect(_localize=False) - variants = variants.flatmap(lambda gvs: hl.rbind( - hl.set({gvs.gene_ids}), - lambda comp_het_gene_ids: gvs.v1.flatmap( - lambda v1: gvs.v2.map(lambda v2: hl.struct( - v1=v1.annotate(comp_het_gene_ids=comp_het_gene_ids), - v2=v2.annotate(comp_het_gene_ids=comp_het_gene_ids), - )) - ) - )) - variants = self._filter_comp_het_families(variants, set_secondary_annotations=False) - return hl.Table.parallelize(variants) - - @classmethod - def _family_filtered_ch_ht(cls, ht, overlapped_families, families): - if overlapped_families == families: - return ht - family_indices = hl.array([families.index(family_guid) for family_guid in overlapped_families]) - ht = ht.annotate(family_entries=family_indices.map(lambda i: ht.family_entries[i])) - return cls._apply_entry_filters(ht, select_globals=False) - - def _is_valid_comp_het_family(self, v1, v2, family_index): - is_valid = super()._is_valid_comp_het_family(v1, v2, family_index) - - # SNPs overlapped by trans deletions may be incorrectly called as hom alt, and should be - # considered comp hets with said deletions. Any other hom alt variants are not valid comp hets - entries_1 = v1.family_entries[family_index] - is_allowed_hom_alt = entries_1.all(lambda g: ~self.GENOTYPE_QUERY_MAP[ALT_ALT](g.GT)) | hl.all([ - v2.sv_type_id == self._sv_type_del_id, - v2.start_locus.position <= v1.locus.position, - v1.locus.position <= v2.end_locus.position, - ]) - return is_valid & is_allowed_hom_alt - - def _comp_het_entry_has_ref(self, gt1, gt2): - variant_query = self._data_type_queries[SNV_INDEL_DATA_TYPE] - sv_query = self._data_type_queries[self._sv_data_type] - return [variant_query.GENOTYPE_QUERY_MAP[REF_REF](gt1), sv_query.GENOTYPE_QUERY_MAP[REF_REF](gt2)] - - def format_search_ht(self): - hts = [] - for data_type, query in self._data_type_queries.items(): - dt_ht = query.format_search_ht() - if dt_ht is None: - continue - dt_ht = self._merged_sort(data_type, dt_ht) - hts.append(dt_ht.select('_sort', **{data_type: dt_ht.row})) - - if self._merged_comp_het_ht is not None: - ch_ht = self._merged_comp_het_ht.annotate( - v1=self._format_comp_het_result(self._merged_comp_het_ht.v1, SNV_INDEL_DATA_TYPE), - v2=self._format_comp_het_result(self._merged_comp_het_ht.v2, self._sv_data_type), - ) - hts.append(ch_ht.select( - _sort=hl.sorted([ch_ht.v1._sort.map(hl.float64), ch_ht.v2._sort.map(hl.float64)])[0], - **{f'comp_het_{self._sv_data_type}': ch_ht.row}, - )) - - if not hts: - return None - - ht = hts[0] - for sub_ht in hts[1:]: - ht = ht.union(sub_ht, unify=True) - - return ht - - def _format_comp_het_result(self, v, data_type): - result = self._data_type_queries[data_type]._format_results(v) - return self._merged_sort(data_type, result) - - def _merged_sort(self, data_type, ht): - # Certain sorts have an extra element for variant-type data, so need to add an element for SV data - if not data_type.startswith('SV'): - return ht - - sort_expr = None - if self._sort == CONSEQUENCE_SORT: - sort_expr = hl.array([hl.float64(4.5)]).extend(ht._sort.map(hl.float64)) - elif self._sort == OMIM_SORT: - sort_expr = hl.array([hl.int64(0)]).extend(ht._sort) - elif self._sort_metadata: - sort_expr = ht._sort[:1].extend(ht._sort) - - if sort_expr is not None: - ht = ht.annotate(_sort=sort_expr) - - return ht - - def _format_collected_rows(self, collected): - data_types = [*self._data_type_queries] - if self._merged_comp_het_ht: - data_types.append(f'comp_het_{self._sv_data_type}') - return super()._format_collected_rows([self._format_collected_row(row, data_types) for row in collected]) - - @staticmethod - def _format_collected_row( row, data_types): - data_type = next(data_type for data_type in data_types if row.get(data_type)) - formatted_row = row.get(data_type) - if 'comp_het' in data_type: - formatted_row = {GROUPED_VARIANTS_FIELD: sorted([formatted_row.v1, formatted_row.v2], key=lambda x: x._sort)} - return formatted_row - - def format_gene_count_hts(self): - hts = [] - for query in self._data_type_queries.values(): - hts += query.format_gene_count_hts() - if self._merged_comp_het_ht is not None: - hts += [ - self._comp_het_gene_count_ht(self._merged_comp_het_ht, 'v1', SNV_INDEL_DATA_TYPE), - self._comp_het_gene_count_ht(self._merged_comp_het_ht, 'v2', self._sv_data_type) - ] - return hts - - def _comp_het_gene_count_ht(self, ht, field, data_type): - selects = { - **self._gene_count_selects(), - 'gene_ids': self._data_type_queries[data_type]._gene_ids_expr, - } - return ht.select(**{k: v(ht[field]) for k, v in selects.items()}) diff --git a/hail_search/queries/snv_indel.py b/hail_search/queries/snv_indel.py deleted file mode 100644 index f83bac64db..0000000000 --- a/hail_search/queries/snv_indel.py +++ /dev/null @@ -1,104 +0,0 @@ -from collections import OrderedDict -import hail as hl - -from hail_search.constants import GENOME_VERSION_GRCh38, SCREEN_KEY, PREFILTER_FREQ_CUTOFF, ALPHAMISSENSE_SORT, \ - UTR_ANNOTATOR_KEY, EXTENDED_SPLICE_KEY, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY, GENOME_VERSION_GRCh37 -from hail_search.queries.base import BaseHailTableQuery, PredictionPath -from hail_search.queries.snv_indel_37 import SnvIndelHailTableQuery37 - -EXTENDED_SPLICE_REGION_CONSEQUENCE = 'extended_intronic_splice_region_variant' - - -class SnvIndelHailTableQuery(SnvIndelHailTableQuery37): - - GENOME_VERSION = GENOME_VERSION_GRCh38 - LIFT_GENOME_VERSION = GENOME_VERSION_GRCh37 - PREDICTION_FIELDS_CONFIG = { - **SnvIndelHailTableQuery37.PREDICTION_FIELDS_CONFIG, - 'fathmm': PredictionPath('dbnsfp', 'fathmm_MKL_coding_score'), - 'mut_pred': PredictionPath('dbnsfp', 'MutPred_score'), - 'vest': PredictionPath('dbnsfp', 'VEST4_score'), - 'gnomad_noncoding': PredictionPath('gnomad_non_coding_constraint', 'z_score'), - } - LIFTOVER_ANNOTATION_FIELDS = BaseHailTableQuery.LIFTOVER_ANNOTATION_FIELDS - ANNOTATION_OVERRIDE_FIELDS = SnvIndelHailTableQuery37.ANNOTATION_OVERRIDE_FIELDS + [ - SCREEN_KEY, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY, - ] - SECONDARY_ANNOTATION_OVERRIDE_FIELDS = [MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY] - FREQUENCY_PREFILTER_FIELDS = OrderedDict([ - (True, 0.001), - ('is_gt_1_percent', PREFILTER_FREQ_CUTOFF), - ('is_gt_3_percent', 0.03), - ('is_gt_5_percent', 0.05), - ('is_gt_10_percent', 0.1), - ]) - SORTS = { - **SnvIndelHailTableQuery37.SORTS, - ALPHAMISSENSE_SORT: lambda r: [ - SnvIndelHailTableQuery37._format_prediction_sort_value( - hl.min(r.sorted_transcript_consequences.map(lambda t: t.alphamissense.pathogenicity)) - ), - SnvIndelHailTableQuery37._format_prediction_sort_value(r.selected_transcript.alphamissense.pathogenicity), - ], - } - - def _get_allowed_consequence_ids(self, annotations): - parsed_allowed_consequences = {} - allowed_consequence_ids = super()._get_allowed_consequence_ids(annotations) - if allowed_consequence_ids: - parsed_allowed_consequences[self.TRANSCRIPT_CONSEQUENCE_FIELD] = allowed_consequence_ids - - utr_consequence_ids = self._get_enum_terms_ids( - self.TRANSCRIPTS_FIELD, subfield='utrannotator', nested_subfield='fiveutr_consequence', - terms=(annotations.get(UTR_ANNOTATOR_KEY) or []), - ) - if utr_consequence_ids: - parsed_allowed_consequences[UTR_ANNOTATOR_KEY] = utr_consequence_ids - - if EXTENDED_SPLICE_REGION_CONSEQUENCE in (annotations.get(EXTENDED_SPLICE_KEY) or []): - parsed_allowed_consequences[EXTENDED_SPLICE_REGION_CONSEQUENCE] = True - - return parsed_allowed_consequences - - @staticmethod - def _get_allowed_transcripts_filter(allowed_consequence_ids): - allowed_consequence_filters = [] - - consequence_ids = allowed_consequence_ids.get(SnvIndelHailTableQuery37.TRANSCRIPT_CONSEQUENCE_FIELD) - if consequence_ids: - allowed_consequence_filters.append(SnvIndelHailTableQuery37._get_allowed_transcripts_filter(consequence_ids)) - - utr_consequences = allowed_consequence_ids.get(UTR_ANNOTATOR_KEY) - if utr_consequences: - utr_consequences = hl.set(utr_consequences) - allowed_consequence_filters.append(lambda tc: utr_consequences.contains(tc.utrannotator.fiveutr_consequence_id)) - - if allowed_consequence_ids.get(EXTENDED_SPLICE_REGION_CONSEQUENCE): - allowed_consequence_filters.append(lambda tc: tc.spliceregion.extended_intronic_splice_region_variant) - - return allowed_consequence_filters[0] if len(allowed_consequence_filters) == 1 else lambda tc: hl.any([ - f(tc) for f in allowed_consequence_filters - ]) - - def _get_annotation_override_filters(self, ht, annotation_overrides): - annotation_filters = super()._get_annotation_override_filters(ht, annotation_overrides) - - if annotation_overrides.get(SCREEN_KEY): - allowed_consequences = hl.set(self._get_enum_terms_ids(SCREEN_KEY.lower(), 'region_type', annotation_overrides[SCREEN_KEY])) - annotation_filters.append(allowed_consequences.contains(ht.screen.region_type_ids.first())) - - for feature_key in [MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY]: - if annotation_overrides.get(feature_key): - field = f'sorted_{feature_key}_consequences' - allowed_consequences = hl.set(self._get_enum_terms_ids( - field, self.TRANSCRIPT_CONSEQUENCE_FIELD, annotation_overrides[feature_key]), - ) - annotation_filters.append( - ht[field].any(lambda c: c.consequence_term_ids.any(allowed_consequences.contains)) - ) - - return annotation_filters - - @staticmethod - def _lookup_variant_annotations(): - return {'liftover_locus': lambda r: r.rg37_locus} diff --git a/hail_search/queries/snv_indel_37.py b/hail_search/queries/snv_indel_37.py deleted file mode 100644 index 31ec40f3fc..0000000000 --- a/hail_search/queries/snv_indel_37.py +++ /dev/null @@ -1,160 +0,0 @@ -from aiohttp.web import HTTPNotFound -from collections import OrderedDict -import hail as hl - -from hail_search.constants import CLINVAR_KEY, HGMD_KEY, HGMD_PATH_RANGES, \ - GNOMAD_GENOMES_FIELD, PREFILTER_FREQ_CUTOFF, PATH_FREQ_OVERRIDE_CUTOFF, PATHOGENICTY_HGMD_SORT_KEY, \ - SPLICE_AI_FIELD, GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38 -from hail_search.queries.base import PredictionPath, QualityFilterFormat -from hail_search.queries.mito import MitoHailTableQuery - - -class SnvIndelHailTableQuery37(MitoHailTableQuery): - - DATA_TYPE = 'SNV_INDEL' - GENOME_VERSION = GENOME_VERSION_GRCh37 - LIFT_GENOME_VERSION = GENOME_VERSION_GRCh38 - - GENOTYPE_FIELDS = {f.lower(): f for f in ['DP', 'GQ', 'AB']} - QUALITY_FILTER_FORMAT = { - 'AB': QualityFilterFormat(override=lambda gt: ~gt.GT.is_het(), scale=100), - } - POPULATIONS = { - 'seqr': {'af': None, 'an': None, 'hom': 'hom', 'hemi': None, 'het': None, 'sort': 'callset_af', 'sort_subfield': 'ac'}, - 'topmed': {'hemi': None}, - 'exac': { - 'filter_af': 'AF_POPMAX', 'ac': 'AC_Adj', 'an': 'AN_Adj', 'hom': 'AC_Hom', 'hemi': 'AC_Hemi', - 'het': 'AC_Het', - }, - 'gnomad_exomes': {'filter_af': 'AF_POPMAX_OR_GLOBAL', 'het': None, 'sort': 'gnomad_exomes'}, - GNOMAD_GENOMES_FIELD: {'filter_af': 'AF_POPMAX_OR_GLOBAL', 'het': None, 'sort': 'gnomad'}, - } - PREDICTION_FIELDS_CONFIG = { - 'cadd': PredictionPath('dbnsfp', 'CADD_phred'), - 'eigen': PredictionPath('eigen', 'Eigen_phred'), - 'mpc': PredictionPath('dbnsfp', 'MPC_score'), - 'primate_ai': PredictionPath('dbnsfp', 'PrimateAI_score'), - SPLICE_AI_FIELD: PredictionPath(SPLICE_AI_FIELD, 'delta_score'), - 'splice_ai_consequence': PredictionPath(SPLICE_AI_FIELD, 'splice_consequence'), - 'mut_taster': PredictionPath('dbnsfp', 'MutationTaster_pred'), - 'polyphen': PredictionPath('dbnsfp', 'Polyphen2_HVAR_score'), - 'revel': PredictionPath('dbnsfp', 'REVEL_score'), - 'sift': PredictionPath('dbnsfp', 'SIFT_score'), - } - PATHOGENICITY_FILTERS = { - **MitoHailTableQuery.PATHOGENICITY_FILTERS, - HGMD_KEY: ('class', HGMD_PATH_RANGES), - } - ANNOTATION_OVERRIDE_FIELDS = [SPLICE_AI_FIELD] - - CORE_FIELDS = MitoHailTableQuery.CORE_FIELDS + ['CAID'] - - LIFTOVER_ANNOTATION_FIELDS = {} - BASE_ANNOTATION_FIELDS = { - k: v for k, v in MitoHailTableQuery.BASE_ANNOTATION_FIELDS.items() - if k not in MitoHailTableQuery.MITO_ANNOTATION_FIELDS - } - ENUM_ANNOTATION_FIELDS = { - **MitoHailTableQuery.ENUM_ANNOTATION_FIELDS, - 'screen': { - 'response_key': 'screenRegionType', - 'format_value': lambda value: value.region_types.first(), - }, - } - - SORTS = { - **MitoHailTableQuery.SORTS, - PATHOGENICTY_HGMD_SORT_KEY: lambda r: [MitoHailTableQuery.CLINVAR_SORT(CLINVAR_KEY, r), r.hgmd.class_id], - } - - FREQUENCY_PREFILTER_FIELDS = OrderedDict([ - (True, PREFILTER_FREQ_CUTOFF), - ('is_gt_10_percent', 0.1), - ]) - - PREFILTER_TABLES = { - **MitoHailTableQuery.PREFILTER_TABLES, - GNOMAD_GENOMES_FIELD: 'high_af_variants.ht', - } - - def _prefilter_entries_table(self, ht, *args, raw_intervals=None, **kwargs): - ht = super()._prefilter_entries_table(ht, *args, **kwargs) - load_table_intervals = self._load_table_kwargs.get('_intervals') or [] - no_interval_prefilter = not load_table_intervals or len(raw_intervals or []) > len(load_table_intervals) - if 'variant_ht' not in self._load_table_kwargs and no_interval_prefilter: - af_ht = self._get_loaded_filter_ht( - GNOMAD_GENOMES_FIELD, self._get_gnomad_af_prefilter, **kwargs) - if af_ht: - ht = ht.filter(hl.is_missing(af_ht[ht.key])) - return ht - - def _get_gnomad_af_prefilter(self, frequencies=None, pathogenicity=None, **kwargs): - gnomad_genomes_filter = (frequencies or {}).get(GNOMAD_GENOMES_FIELD, {}) - af_cutoff = gnomad_genomes_filter.get('af') - if af_cutoff is None and gnomad_genomes_filter.get('ac') is not None: - af_cutoff = PREFILTER_FREQ_CUTOFF - if af_cutoff is None: - return False - - af_cutoff_field = self._get_af_prefilter_field(af_cutoff) - if af_cutoff_field is None: - return False - - clinvar_path_ht = False - if af_cutoff < PATH_FREQ_OVERRIDE_CUTOFF: - clinvar_path_ht = self._get_loaded_clinvar_prefilter_ht(pathogenicity) - - if clinvar_path_ht is not False: - path_cutoff_field = self._get_af_prefilter_field(PATH_FREQ_OVERRIDE_CUTOFF) - non_clinvar_filter = lambda ht: hl.is_missing(clinvar_path_ht[ht.key]) - if af_cutoff_field is not True: - non_clinvar_var_filter = non_clinvar_filter - non_clinvar_filter = lambda ht: non_clinvar_var_filter(ht) & self._af_prefilter(af_cutoff_field)(ht) - af_filter = lambda ht: ht[path_cutoff_field] | non_clinvar_filter(ht) - else: - af_filter = self._af_prefilter(af_cutoff_field) - - return af_filter - - @staticmethod - def _af_prefilter(af_cutoff_field): - return True if af_cutoff_field is True else lambda ht: ht[af_cutoff_field] - - def _get_af_prefilter_field(self, af_cutoff): - return next((field for field, cutoff in self.FREQUENCY_PREFILTER_FIELDS.items() if af_cutoff <= cutoff), None) - - def _get_annotation_override_filters(self, ht, annotation_overrides): - annotation_filters = super()._get_annotation_override_filters(ht, annotation_overrides) - - if annotation_overrides.get(SPLICE_AI_FIELD): - score_filter, _ = self._get_in_silico_filter(ht, SPLICE_AI_FIELD, annotation_overrides[SPLICE_AI_FIELD]) - annotation_filters.append(score_filter) - - return annotation_filters - - @staticmethod - def _stat_has_non_ref(s): - return (s.het_samples > 0) | (s.hom_samples > 0) - - @staticmethod - def _lookup_variant_annotations(): - return {'liftover_locus': lambda r: r.rg38_locus} - - @classmethod - def _get_lifted_table_path(cls, path): - return f'{cls._get_table_dir(path)}/{cls.LIFT_GENOME_VERSION}/{cls.DATA_TYPE}/{path}' - - def _get_variant_project_data(self, variant_id, variant=None, **kwargs): - project_data = super()._get_variant_project_data(variant_id, **kwargs) - liftover_locus = variant.pop('liftover_locus') - if not liftover_locus: - return project_data - interval = hl.eval(hl.interval(liftover_locus, liftover_locus, includes_start=True, includes_end=True)) - self._load_table_kwargs['_intervals'] = [interval] - self._get_table_path = self._get_lifted_table_path - try: - lift_project_data = super()._get_variant_project_data(variant_id, **kwargs) - except HTTPNotFound: - return project_data - project_data['familyGenotypes'].update(lift_project_data['familyGenotypes']) - return project_data.annotate(liftedFamilyGuids=sorted(lift_project_data['familyGenotypes'].keys())) diff --git a/hail_search/queries/sv.py b/hail_search/queries/sv.py deleted file mode 100644 index 01df7afc57..0000000000 --- a/hail_search/queries/sv.py +++ /dev/null @@ -1,172 +0,0 @@ -import hail as hl - - -from hail_search.constants import CONSEQUENCE_SORT, NEW_SV_FIELD, STRUCTURAL_ANNOTATION_FIELD, FAMILY_GUID_FIELD, \ - GENOTYPES_FIELD -from hail_search.queries.base import BaseHailTableQuery, PredictionPath - - -class SvHailTableQuery(BaseHailTableQuery): - - DATA_TYPE = 'SV_WGS' - KEY_FIELD = ('variant_id',) - - GENOTYPE_FIELDS = {f.lower(): f for f in ['CN', 'GQ']} - COMPUTED_GENOTYPE_FIELDS = { - k: lambda entry, field, *args: entry.concordance[field] for k in ['new_call', 'prev_call', 'prev_num_alt'] - } - GENOTYPE_QUERY_FIELDS = {'gq_sv': 'GQ', 'gq': None} - - TRANSCRIPTS_FIELD = 'sorted_gene_consequences' - TRANSCRIPT_CONSEQUENCE_FIELD = 'major_consequence' - CORE_FIELDS = BaseHailTableQuery.CORE_FIELDS + ['algorithms'] - BASE_ANNOTATION_FIELDS = { - 'bothsidesSupport': lambda r: r.bothsides_support, - 'chrom': lambda r: r.start_locus.contig.replace('^chr', ''), - 'pos': lambda r: r.start_locus.position, - 'end': lambda r: r.end_locus.position, - 'rg37LocusEnd': lambda r: hl.or_missing( - hl.is_defined(r.rg37_locus_end), hl.struct(contig=r.rg37_locus_end.contig, position=r.rg37_locus_end.position) - ), - **BaseHailTableQuery.BASE_ANNOTATION_FIELDS, - } - ENUM_ANNOTATION_FIELDS = { - TRANSCRIPTS_FIELD: BaseHailTableQuery.ENUM_ANNOTATION_FIELDS['transcripts'], - } - - POPULATIONS = { - 'sv_callset': {'hemi': None, 'sort': 'callset_af', 'sort_subfield': 'ac'}, - 'gnomad_svs': {'id': 'ID', 'ac': None, 'an': None, 'hom': 'N_HOM', 'hemi': None, 'het': 'N_HET', 'sort': 'gnomad'}, - } - POPULATION_FIELDS = {'sv_callset': 'gt_stats'} - PREDICTION_FIELDS_CONFIG = { - 'strvctvre': PredictionPath('strvctvre', 'score'), - } - ANNOTATION_OVERRIDE_FIELDS = [STRUCTURAL_ANNOTATION_FIELD] - SECONDARY_ANNOTATION_OVERRIDE_FIELDS = [STRUCTURAL_ANNOTATION_FIELD] - - SORTS = { - **BaseHailTableQuery.SORTS, - CONSEQUENCE_SORT: lambda r: [hl.min(r.sorted_gene_consequences.map(lambda g: g.major_consequence_id))], - 'size': lambda r: [hl.if_else( - r.start_locus.contig == r.end_locus.contig, r.start_locus.position - r.end_locus.position, -50, - )], - } - - def __init__(self, *args, padded_interval=None, **kwargs): - self._is_interval_filtered = bool(padded_interval) - super().__init__(*args, padded_interval=padded_interval, **kwargs) - - def _set_interval_prefilter(self, *args, **kwargs): - self._is_interval_filtered = True - - @classmethod - def _get_sample_type(cls, *args): - return cls.DATA_TYPE.split('_')[-1] - - def _read_project_table(self, project_guid: str, sample_type: str, **kwargs): - ht = super()._read_project_table(project_guid, sample_type, **kwargs) - if ht is not None: - ht = ht.annotate_globals(sample_type=sample_type) - return ht - - def import_filtered_table(self, project_samples, *args, **kwargs): - if len(project_samples) == 1 or not self._is_interval_filtered: - return super().import_filtered_table(project_samples, *args, **kwargs) - - ht = self._read_table('annotations.ht') - ht = self._filter_annotated_table(ht, is_comp_het=self._has_comp_het_search, **kwargs) - - self._load_table_kwargs['variant_ht'] = ht.select() - families_ht, comp_het_families_ht = self._import_families_tables(project_samples, *args, skip_missing_field='family_entries', **kwargs) - - if comp_het_families_ht is not None: - self._comp_het_ht = self._annotate_families_table_annotations(comp_het_families_ht, ht, is_comp_het=True) - self._comp_het_ht = self._filter_compound_hets() - - if families_ht is not None: - self._ht = self._annotate_families_table_annotations(families_ht, ht) - if self._has_secondary_annotations: - self._ht = self._ht.filter(hl.any(self._get_annotation_filters(self._ht))) - - def _annotate_families_table_annotations(self, families_ht, annotations_ht, is_comp_het=False): - return families_ht.annotate(**annotations_ht[families_ht.key]) - - def _filter_annotated_table(self, ht, *args, parsed_intervals=None, exclude_intervals=False, padded_interval=None, gene_ids=None, **kwargs): - if parsed_intervals and len(parsed_intervals) != len(gene_ids or []): - interval_filter = hl.array(parsed_intervals).any(lambda interval: hl.if_else( - ht.start_locus.contig == ht.end_locus.contig, - interval.overlaps(hl.interval(ht.start_locus, ht.end_locus)), - interval.contains(ht.start_locus) | interval.contains(ht.end_locus), - )) - if exclude_intervals: - interval_filter = ~interval_filter - ht = ht.filter(interval_filter) - if padded_interval: - padding = int((padded_interval['end'] - padded_interval['start']) * padded_interval['padding']) - chrom = f"chr{padded_interval['chrom']}" - ht = ht.filter(hl.all([ - self._locus_in_range(ht.start_locus, chrom, padded_interval['start'], padding), - self._locus_in_range(ht.end_locus, chrom, padded_interval['end'], padding) - ])) - - return super()._filter_annotated_table(ht, *args, gene_ids=gene_ids, **kwargs) - - def _locus_in_range(self, locus, chrom, position, padding): - window = hl.locus(chrom, position, reference_genome=self.GENOME_VERSION).window(padding, padding) - return window.contains(locus) - - def _parse_annotations(self, annotations, *args, **kwargs): - parsed_annotations = super()._parse_annotations(annotations, *args, **kwargs) - parsed_annotations[NEW_SV_FIELD] = (annotations or {}).get(NEW_SV_FIELD) - return parsed_annotations - - def _get_family_passes_quality_filter(self, quality_filter, ht, parsed_annotations=None, **kwargs): - passes_quality = super()._get_family_passes_quality_filter(quality_filter, ht) - if not (parsed_annotations or {}).get(NEW_SV_FIELD): - return passes_quality - - entries_has_new_call = lambda entries: entries.any(lambda x: x.concordance.new_call) - if passes_quality is None: - return entries_has_new_call - - return lambda entries: entries_has_new_call(entries) & passes_quality(entries) - - def _get_annotation_override_filters(self, ht, annotation_overrides): - annotation_filters = [] - if annotation_overrides.get(STRUCTURAL_ANNOTATION_FIELD): - allowed_type_ids = self.get_allowed_sv_type_ids(annotation_overrides[STRUCTURAL_ANNOTATION_FIELD]) - if allowed_type_ids: - annotation_filters.append(hl.set(allowed_type_ids).contains(ht.sv_type_id)) - - return annotation_filters - - def get_allowed_sv_type_ids(self, sv_types): - return self._get_enum_terms_ids('sv_type', None, sv_types) - - def _additional_annotation_fields(self): - sv_type_enum = self._enums['sv_type'] - insertion_type_id = sv_type_enum.index('INS') - get_end_chrom = lambda r: hl.or_missing(r.start_locus.contig != r.end_locus.contig, r.end_locus.contig.replace('^chr', '')) - return { - 'cpxIntervals': lambda r: self._format_enum( - r, 'cpx_intervals', {'type': sv_type_enum}, annotate_value=lambda val, *args: { - 'chrom': val.start.contig, - 'start': val.start.position, - 'end': val.end.position, - }, - ), - # For insertions, end_locus represents the svSourceDetail, otherwise represents the endChrom - 'endChrom': lambda r: hl.or_missing(r.sv_type_id != insertion_type_id, get_end_chrom(r)), - 'svSourceDetail': lambda r: hl.or_missing(r.sv_type_id == insertion_type_id, hl.bind( - lambda end_chrom: hl.or_missing(hl.is_defined(end_chrom), hl.struct(chrom=end_chrom)), - get_end_chrom(r), - )), - } - - def _import_variant_projects_ht(self, variant_id, sample_data=None, **kwargs): - project_samples, _ = self._parse_sample_data(sample_data) - projects_ht = super()._import_variant_projects_ht(variant_id, project_samples=project_samples) - - annotation_fields = self.annotation_fields(include_genotype_overrides=False) - return projects_ht.select(**{k: annotation_fields[k](projects_ht) for k in [FAMILY_GUID_FIELD, GENOTYPES_FIELD]}) diff --git a/hail_search/requirements-test.in b/hail_search/requirements-test.in deleted file mode 100644 index 245d0d3637..0000000000 --- a/hail_search/requirements-test.in +++ /dev/null @@ -1,2 +0,0 @@ -coverage<5.2 -pytest-aiohttp diff --git a/hail_search/requirements-test.txt b/hail_search/requirements-test.txt deleted file mode 100644 index fa1e1672a2..0000000000 --- a/hail_search/requirements-test.txt +++ /dev/null @@ -1,50 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# To update, run: -# -# pip-compile hail_search/requirements-test.in -# -aiohappyeyeballs==2.3.5 - # via aiohttp -aiohttp==3.10.11 - # via pytest-aiohttp -aiosignal==1.3.1 - # via aiohttp -async-timeout==4.0.2 - # via aiohttp -attrs==23.1.0 - # via aiohttp -coverage==5.1 - # via -r requirements-test.in -exceptiongroup==1.1.3 - # via pytest -frozenlist==1.3.3 - # via - # aiohttp - # aiosignal -idna==3.7 - # via yarl -iniconfig==2.0.0 - # via pytest -multidict==6.0.4 - # via - # aiohttp - # yarl -packaging==23.1 - # via pytest -pluggy==1.2.0 - # via pytest -propcache==0.2.0 - # via yarl -pytest==7.4.0 - # via - # pytest-aiohttp - # pytest-asyncio -pytest-aiohttp==1.0.4 - # via -r requirements-test.in -pytest-asyncio==0.21.0 - # via pytest-aiohttp -tomli==2.0.1 - # via pytest -yarl==1.17.2 - # via aiohttp diff --git a/hail_search/search.py b/hail_search/search.py deleted file mode 100644 index 8f43137191..0000000000 --- a/hail_search/search.py +++ /dev/null @@ -1,39 +0,0 @@ -from hail_search.queries.multi_data_types import QUERY_CLASS_MAP, SNV_INDEL_DATA_TYPE, MultiDataTypeHailTableQuery - - -def search_hail_backend(request, gene_counts=False): - sample_data = request.pop('sample_data', {}) - genome_version = request.pop('genome_version') - - data_types = list(sample_data.keys()) - single_data_type = data_types[0] if len(data_types) == 1 else None - - if single_data_type: - sample_data = sample_data[single_data_type] - query_cls = QUERY_CLASS_MAP[(single_data_type, genome_version)] - else: - query_cls = MultiDataTypeHailTableQuery - - query = query_cls(sample_data, **request) - if gene_counts: - return query.gene_counts() - else: - return query.search() - - -def lookup_variant(request): - data_type = request.get('data_type', SNV_INDEL_DATA_TYPE) - query = QUERY_CLASS_MAP[(data_type, request['genome_version'])](sample_data=None) - return query.lookup_variant(request['variant_id'], sample_data=request.get('sample_data')) - - -def lookup_variants(request): - query = QUERY_CLASS_MAP[(request['data_type'], request['genome_version'])](sample_data=None) - return query.lookup_variants(request['variant_ids']) - - -def load_globals(): - return { - str(k): v.load_globals() - for k, v in QUERY_CLASS_MAP.items() - } diff --git a/hail_search/test_search.py b/hail_search/test_search.py deleted file mode 100644 index cd3235e79c..0000000000 --- a/hail_search/test_search.py +++ /dev/null @@ -1,1293 +0,0 @@ -from aiohttp.test_utils import AioHTTPTestCase -import asyncio -from copy import deepcopy -import hail as hl -import time -from unittest import mock - -from hail_search.test_utils import get_hail_search_body, FAMILY_2_VARIANT_SAMPLE_DATA, FAMILY_2_MISSING_SAMPLE_DATA, \ - VARIANT1, VARIANT2, VARIANT3, VARIANT4, MULTI_PROJECT_SAMPLE_DATA, MULTI_PROJECT_MISSING_SAMPLE_DATA, \ - LOCATION_SEARCH, EXCLUDE_LOCATION_SEARCH, VARIANT_ID_SEARCH, RSID_SEARCH, GENE_COUNTS, SV_WGS_SAMPLE_DATA, \ - SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, \ - GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, SV_WES_SAMPLE_DATA, EXPECTED_SAMPLE_DATA, \ - FAMILY_2_MITO_SAMPLE_DATA, FAMILY_2_ALL_SAMPLE_DATA, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3, \ - EXPECTED_SAMPLE_DATA_WITH_SEX, SV_WGS_SAMPLE_DATA_WITH_SEX, VARIANT_LOOKUP_VARIANT, \ - MULTI_PROJECT_SAMPLE_TYPES_SAMPLE_DATA, FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA, \ - VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA_MISSING_PARENTAL_WGS, \ - VARIANT4_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, \ - VARIANT1_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, \ - VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, VARIANT3_BOTH_SAMPLE_TYPES, PROJECT_2_VARIANT, GRCH37_VARIANT -from hail_search.web_app import init_web_app, sync_to_async_hail_query -from hail_search.queries.base import BaseHailTableQuery - -PROJECT_2_VARIANT_BOTH_SAMPLE_TYPES = deepcopy(PROJECT_2_VARIANT) -PROJECT_2_VARIANT_BOTH_SAMPLE_TYPES['genotypes']['I000015_na20885'] = { - 'sampleId': 'NA20885', 'sampleType': 'WGS', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'dp': 8, 'gq': 14, 'ab': 0.875, 'filters': [], -} - -FAMILY_3_VARIANT = deepcopy(VARIANT3) -FAMILY_3_VARIANT['familyGuids'] = ['F000003_3'] -FAMILY_3_VARIANT['genotypes'] = { - 'I000007_na20870': { - 'sampleId': 'NA20870', 'sampleType': 'WES', 'individualGuid': 'I000007_na20870', 'familyGuid': 'F000003_3', - 'numAlt': 1, 'dp': 28, 'gq': 99, 'ab': 0.6785714285714286, 'filters': [], - }, -} - -MULTI_FAMILY_VARIANT = deepcopy(VARIANT3) -MULTI_FAMILY_VARIANT['familyGuids'] += FAMILY_3_VARIANT['familyGuids'] -MULTI_FAMILY_VARIANT['genotypes'].update(FAMILY_3_VARIANT['genotypes']) - -SELECTED_TRANSCRIPT_MULTI_FAMILY_VARIANT = {**MULTI_FAMILY_VARIANT, 'selectedMainTranscriptId': 'ENST00000426137'} -SELECTED_ANNOTATION_TRANSCRIPT_MULTI_FAMILY_VARIANT = {**MULTI_FAMILY_VARIANT, 'selectedMainTranscriptId': 'ENST00000497611'} -SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4 = {**VARIANT4, 'selectedMainTranscriptId': 'ENST00000350997'} -SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_3 = {**VARIANT3, 'selectedMainTranscriptId': 'ENST00000497611'} -SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2 = {**VARIANT2, 'selectedMainTranscriptId': 'ENST00000459627'} -MULTI_DATA_TYPE_COMP_HET_VARIANT2 = {**VARIANT2, 'selectedMainTranscriptId': 'ENST00000450625'} - -PROJECT_2_VARIANT1 = deepcopy(VARIANT1) -PROJECT_2_VARIANT1['familyGuids'] = ['F000011_11'] -PROJECT_2_VARIANT1['genotypes'] = { - 'I000015_na20885': { - 'sampleId': 'NA20885', 'sampleType': 'WES', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 2, 'dp': 6, 'gq': 16, 'ab': 1.0, 'filters': [], - }, -} -MULTI_PROJECT_VARIANT1 = deepcopy(VARIANT1) -MULTI_PROJECT_VARIANT1['familyGuids'] += PROJECT_2_VARIANT1['familyGuids'] -MULTI_PROJECT_VARIANT1['genotypes'].update(PROJECT_2_VARIANT1['genotypes']) -MULTI_PROJECT_VARIANT2 = deepcopy(VARIANT2) -MULTI_PROJECT_VARIANT2['familyGuids'].append('F000011_11') -MULTI_PROJECT_VARIANT2['genotypes']['I000015_na20885'] = { - 'sampleId': 'NA20885', 'sampleType': 'WES', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'dp': 28, 'gq': 99, 'ab': 0.5, 'filters': [], -} - -NO_GENOTYPE_GCNV_VARIANT = {**GCNV_VARIANT4, 'numExon': 8, 'end': 38736268} - -# Ensures no variants are filtered out by annotation/path filters for compound hets -COMP_HET_ALL_PASS_FILTERS = { - 'annotations': {'splice_ai': '0.0', 'structural': ['DEL', 'CPX', 'INS', 'gCNV_DEL', 'gCNV_DUP']}, - 'pathogenicity': {'clinvar': ['likely_pathogenic']}, -} - -NEW_SV_FILTER = {'new_structural_variants': ['NEW']} - -SV_GENE_COUNTS = { - 'ENSG00000171621': {'total': 2, 'families': {'F000011_11': 2}}, - 'ENSG00000083544': {'total': 1, 'families': {'F000011_11': 1}}, - 'ENSG00000184986': {'total': 1, 'families': {'F000011_11': 1}}, - 'null': {'total': 1, 'families': {'F000011_11': 1}}, -} - -GCNV_GENE_COUNTS = { - 'ENSG00000103495': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000167371': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000280893': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000275023': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000277972': {'total': 1, 'families': {'F000002_2': 1}}, -} - -OMIM_SORT_METADATA = ['ENSG00000177000', 'ENSG00000097046', 'ENSG00000275023'] - -MULTI_PROJECT_BOTH_SAMPLE_TYPE_VARIANTS = [ - deepcopy(v) for v in - [PROJECT_2_VARIANT, MULTI_PROJECT_VARIANT1, MULTI_PROJECT_VARIANT2, VARIANT3, VARIANT4] -] -for v in MULTI_PROJECT_BOTH_SAMPLE_TYPE_VARIANTS: - for indiv_id, gts in v['genotypes'].items(): - v['genotypes'][indiv_id] = [gts] -for v in MULTI_PROJECT_BOTH_SAMPLE_TYPE_VARIANTS[:-2]: - v['genotypes']['I000015_na20885'].append({**v['genotypes']['I000015_na20885'][0], 'sampleType': 'WGS'}) - -SECOND_PROJECT_SV_WES_SAMPLE_DATA = [{**s, 'sample_type': 'WES'} for s in SV_WGS_SAMPLE_DATA['SV_WGS']] - -MULTI_PROJECT_GCNV_VARIANT3 = { - **GCNV_VARIANT3, - 'familyGuids': GCNV_VARIANT3['familyGuids'] + ['F000011_11'], - 'genotypes': { - **GCNV_VARIANT3['genotypes'], - 'I000015_na20885': { - 'sampleId': 'NA20885', 'sampleType': 'WES', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 2, 'cn': 4, 'qs': 27, 'defragged': True, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': True, 'prevCall': False, 'prevOverlap': False, 'filters': [], - }, - 'I000035_na20883': { - 'sampleId': 'NA20883', 'sampleType': 'WES', 'individualGuid': 'I000035_na20883', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'cn': 3, 'qs': 51, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': False, 'prevOverlap': True, 'filters': [], - }, - 'I000025_na20884': { - 'sampleId': 'NA20884', 'sampleType': 'WES', 'individualGuid': 'I000025_na20884', 'familyGuid': 'F000011_11', 'numAlt': 0, - 'cn': None, 'qs': None, 'defragged': None, 'start': None, 'end': None, 'numExon': None, 'geneIds': None, - 'newCall': None, 'prevCall': None, 'prevOverlap': None, 'filters': [], - }, - }, -} - -def _sorted(variant, sorts): - return {**variant, '_sort': sorts + variant['_sort']} - - -class HailSearchTestCase(AioHTTPTestCase): - - async def get_application(self): - return await init_web_app() - - async def test_sync_to_async_hail_query(self): - request = mock.Mock() - request.app = await self.get_application() - # NB: request.json() is the first arg passed to the callable - request.json.return_value = asyncio.Future() - request.json.return_value.set_result(3) - with self.assertRaises(TimeoutError): - await sync_to_async_hail_query(request, time.sleep, timeout_s=1) - - with mock.patch('hail_search.web_app.ctypes.pythonapi.PyThreadState_SetAsyncExc') as mock_set_async_exc: - mock_set_async_exc.return_value = 2 - with self.assertRaises(SystemExit): - await sync_to_async_hail_query(request, time.sleep, timeout_s=1) - - async def test_status(self): - async with self.client.request('GET', '/status') as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertDictEqual(resp_json, {'success': True}) - - async def test_reload_globals(self): - async with self.client.request('POST', '/reload_globals') as resp: - resp_json = await resp.json() - self.assertTrue( - resp_json["('SNV_INDEL', 'GRCh38')"]['versions']['gnomad_genomes'], - ) - with mock.patch('hail_search.queries.base.hl.read_table') as mock_read_table: - mock_read_table.return_value = hl.Table.parallelize( - [], - hl.tstruct(), - globals=hl.Struct( - enums=hl.Struct(reloaded_enum=1), - versions=hl.Struct(reloaded_version=2), - ) - ) - async with self.client.request('POST', '/reload_globals') as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertDictEqual( - resp_json["('SNV_INDEL', 'GRCh38')"], - { - 'enums': {'reloaded_enum': 1}, - 'versions': {'reloaded_version': 2}, - }, - ) - - async def _assert_expected_search(self, results, gene_counts=None, **search_kwargs): - search_body = get_hail_search_body(**search_kwargs) - async with self.client.request('POST', '/search', json=search_body) as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertSetEqual(set(resp_json.keys()), {'results', 'total'}) - self.assertEqual(resp_json['total'], len(results)) - for i, result in enumerate(resp_json['results']): - self.assertEqual(result, results[i]) - - if gene_counts: - async with self.client.request('POST', '/gene_counts', json=search_body) as resp: - self.assertEqual(resp.status, 200) - gene_counts_json = await resp.json() - self.assertDictEqual(gene_counts_json, gene_counts) - - async def test_single_family_search(self): - variant_gene_counts = { - 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}} - } - await self._assert_expected_search( - [VARIANT1, VARIANT2, VARIANT3, VARIANT4], sample_data=FAMILY_2_VARIANT_SAMPLE_DATA, gene_counts=variant_gene_counts, - ) - - mito_gene_counts = { - 'ENSG00000210112': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000198886': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000198727': {'total': 1, 'families': {'F000002_2': 1}}, - } - await self._assert_expected_search( - [MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sample_data=FAMILY_2_MITO_SAMPLE_DATA, gene_counts=mito_gene_counts, - ) - - await self._assert_expected_search( - [GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], omit_data_type='SNV_INDEL', gene_counts=GCNV_GENE_COUNTS, - ) - - await self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4], sample_data=SV_WGS_SAMPLE_DATA, gene_counts=SV_GENE_COUNTS, - ) - - await self._assert_expected_search( - [VARIANT1, SV_VARIANT1, SV_VARIANT2, VARIANT2, VARIANT3, VARIANT4, SV_VARIANT3, GCNV_VARIANT1, SV_VARIANT4, - GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], sample_data={ - 'SV_WES': EXPECTED_SAMPLE_DATA['SV_WES'], **FAMILY_2_ALL_SAMPLE_DATA, **SV_WGS_SAMPLE_DATA, - }, gene_counts={**variant_gene_counts, **mito_gene_counts, **GCNV_GENE_COUNTS, **SV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}}, - ) - - await self._assert_expected_search( - [GRCH37_VARIANT], genome_version='GRCh37', sample_data=FAMILY_2_VARIANT_SAMPLE_DATA) - - async def test_single_project_search(self): - variant_gene_counts = { - 'ENSG00000097046': {'total': 3, 'families': {'F000002_2': 2, 'F000003_3': 1}}, - 'ENSG00000177000': {'total': 3, 'families': {'F000002_2': 2, 'F000003_3': 1}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - } - await self._assert_expected_search( - [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4], omit_data_type='SV_WES', gene_counts=variant_gene_counts, - ) - - await self._assert_expected_search( - [GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], sample_data=SV_WES_SAMPLE_DATA, gene_counts={ - 'ENSG00000129562': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000013364': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000079616': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000103495': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000167371': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000280789': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000280893': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000281348': {'total': 2, 'families': {'F000002_2': 1, 'F000003_3': 1}}, - 'ENSG00000275023': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000277972': {'total': 1, 'families': {'F000002_2': 1}}, - } - ) - - await self._assert_expected_search( - [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - gene_counts={**variant_gene_counts, **GCNV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}} - ) - - async def test_multi_project_search(self): - await self._assert_expected_search( - [PROJECT_2_VARIANT, MULTI_PROJECT_VARIANT1, MULTI_PROJECT_VARIANT2, VARIANT3, VARIANT4], - gene_counts=GENE_COUNTS, sample_data=MULTI_PROJECT_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [PROJECT_2_VARIANT, MULTI_PROJECT_VARIANT1, SV_VARIANT1, SV_VARIANT2, MULTI_PROJECT_VARIANT2, VARIANT3, - VARIANT4, SV_VARIANT3, SV_VARIANT4], gene_counts={**GENE_COUNTS, **SV_GENE_COUNTS}, - sample_data={**MULTI_PROJECT_SAMPLE_DATA, **SV_WGS_SAMPLE_DATA}, - ) - - async def test_both_sample_types_search(self): - # One family (F000011_11) in a multi-project search has identical exome and genome data. - await self._assert_expected_search( - MULTI_PROJECT_BOTH_SAMPLE_TYPE_VARIANTS, gene_counts=GENE_COUNTS, sample_data=MULTI_PROJECT_SAMPLE_TYPES_SAMPLE_DATA, - ) - - # Variant 1 is de novo in exome but inherited and homozygous in genome. - # Variant 2 is inherited and homozygous in exome and de novo and homozygous in genome. - # Variant 3 is inherited in both sample types. - # Variant 4 is de novo in exome, but inherited in genome in the same parent that has variant 3. - inheritance_mode = 'recessive' - await self._assert_expected_search( - [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, [VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES]], - sample_data=FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA, inheritance_mode=inheritance_mode, - **COMP_HET_ALL_PASS_FILTERS - ) - await self._assert_expected_search( - [VARIANT1_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, VARIANT2_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, - [VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY]], - sample_data=FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA_MISSING_PARENTAL_WGS, inheritance_mode=inheritance_mode, - **COMP_HET_ALL_PASS_FILTERS - ) - - inheritance_mode = 'de_novo' - await self._assert_expected_search( - [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES], - sample_data=FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA, inheritance_mode=inheritance_mode, - ) - # Variant 2 fails inheritance when parental data is missing in genome - await self._assert_expected_search( - [VARIANT1_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY, VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY], - sample_data=FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA_MISSING_PARENTAL_WGS, inheritance_mode=inheritance_mode, - ) - - async def test_inheritance_filter(self): - inheritance_mode = 'any_affected' - await self._assert_expected_search( - [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - inheritance_mode=inheritance_mode, - ) - - await self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT2, SV_VARIANT3, SV_VARIANT4], inheritance_mode=inheritance_mode, sample_data=SV_WGS_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [GCNV_VARIANT3], inheritance_mode=inheritance_mode, annotations=NEW_SV_FILTER, omit_data_type='SNV_INDEL', - ) - - await self._assert_expected_search( - [SV_VARIANT2], inheritance_mode=inheritance_mode, annotations=NEW_SV_FILTER, sample_data=SV_WGS_SAMPLE_DATA, - ) - - inheritance_mode = 'de_novo' - await self._assert_expected_search( - [VARIANT1, FAMILY_3_VARIANT, VARIANT4, GCNV_VARIANT1], inheritance_mode=inheritance_mode, - ) - - await self._assert_expected_search( - [SV_VARIANT1], inheritance_mode=inheritance_mode, sample_data=SV_WGS_SAMPLE_DATA, - ) - - inheritance_mode = 'x_linked_recessive' - await self._assert_expected_search([], inheritance_mode=inheritance_mode, sample_data=EXPECTED_SAMPLE_DATA_WITH_SEX) - await self._assert_expected_search([], inheritance_mode=inheritance_mode, sample_data=SV_WGS_SAMPLE_DATA_WITH_SEX) - - inheritance_mode = 'homozygous_recessive' - await self._assert_expected_search([VARIANT2, GCNV_VARIANT3], inheritance_mode=inheritance_mode) - - await self._assert_expected_search( - [PROJECT_2_VARIANT1, VARIANT2], inheritance_mode=inheritance_mode, sample_data=MULTI_PROJECT_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [SV_VARIANT4], inheritance_mode=inheritance_mode, sample_data=SV_WGS_SAMPLE_DATA, - ) - - gt_inheritance_filter = {'genotype': {'I000006_hg00733': 'ref_ref', 'I000005_hg00732': 'has_alt'}} - await self._assert_expected_search( - [VARIANT2], inheritance_filter=gt_inheritance_filter, sample_data=FAMILY_2_VARIANT_SAMPLE_DATA) - - inheritance_mode = 'compound_het' - await self._assert_expected_search( - [[VARIANT3, VARIANT4]], inheritance_mode=inheritance_mode, sample_data=MULTI_PROJECT_SAMPLE_DATA, gene_counts={ - 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 1, 'families': {'F000002_2': 1}}, - }, **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode=inheritance_mode, omit_data_type='SNV_INDEL', gene_counts={ - 'ENSG00000275023': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000277972': {'total': 1, 'families': {'F000002_2': 1}}, - }, **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], [GCNV_VARIANT3, GCNV_VARIANT4]], - inheritance_mode=inheritance_mode, gene_counts={ - 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000275023': {'total': 3, 'families': {'F000002_2': 3}}, - 'ENSG00000277258': {'total': 3, 'families': {'F000002_2': 3}}, - 'ENSG00000277972': {'total': 2, 'families': {'F000002_2': 2}}, - }, **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[SV_VARIANT1, SV_VARIANT2]], inheritance_mode=inheritance_mode, sample_data=SV_WGS_SAMPLE_DATA, - **COMP_HET_ALL_PASS_FILTERS, gene_counts={'ENSG00000171621': {'total': 2, 'families': {'F000011_11': 2}}}, - ) - - await self._assert_expected_search( - [[SV_VARIANT1, SV_VARIANT2], [VARIANT3, VARIANT4]], inheritance_mode=inheritance_mode, - sample_data={**SV_WGS_SAMPLE_DATA, **FAMILY_2_VARIANT_SAMPLE_DATA}, **COMP_HET_ALL_PASS_FILTERS, gene_counts={ - 'ENSG00000171621': {'total': 2, 'families': {'F000011_11': 2}}, - 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 1, 'families': {'F000002_2': 1}}, - }, - ) - - inheritance_mode = 'recessive' - await self._assert_expected_search( - [PROJECT_2_VARIANT1, VARIANT2, [VARIANT3, VARIANT4]], inheritance_mode=inheritance_mode, gene_counts={ - 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - }, sample_data=MULTI_PROJECT_SAMPLE_DATA, **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode=inheritance_mode, omit_data_type='SNV_INDEL', gene_counts={ - 'ENSG00000275023': {'total': 3, 'families': {'F000002_2': 3}}, - 'ENSG00000277258': {'total': 1, 'families': {'F000002_2': 1}}, - 'ENSG00000277972': {'total': 1, 'families': {'F000002_2': 1}}, - }, **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], - inheritance_mode=inheritance_mode, gene_counts={ - 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 3, 'families': {'F000002_2': 3}}, - 'ENSG00000275023': {'total': 4, 'families': {'F000002_2': 4}}, - 'ENSG00000277258': {'total': 4, 'families': {'F000002_2': 4}}, - 'ENSG00000277972': {'total': 2, 'families': {'F000002_2': 2}}, - }, **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[SV_VARIANT1, SV_VARIANT2], SV_VARIANT4], inheritance_mode=inheritance_mode, sample_data=SV_WGS_SAMPLE_DATA, - **COMP_HET_ALL_PASS_FILTERS, gene_counts={ - 'ENSG00000171621': {'total': 2, 'families': {'F000011_11': 2}}, - 'ENSG00000184986': {'total': 1, 'families': {'F000011_11': 1}}, - } - ) - - async def test_quality_filter(self): - quality_filter = {'vcf_filter': 'pass'} - await self._assert_expected_search( - [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - quality_filter=quality_filter - ) - - await self._assert_expected_search( - [SV_VARIANT4, MITO_VARIANT1, MITO_VARIANT2], quality_filter=quality_filter, - sample_data={**SV_WGS_SAMPLE_DATA, **FAMILY_2_MITO_SAMPLE_DATA} - ) - - await self._assert_expected_search( - [MITO_VARIANT1, MITO_VARIANT3], quality_filter={'min_gq': 60, 'min_hl': 5}, sample_data=FAMILY_2_MITO_SAMPLE_DATA, - ) - - gcnv_quality_filter = {'min_gq': 40, 'min_qs': 20} - await self._assert_expected_search( - [VARIANT2, MULTI_FAMILY_VARIANT, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT4], quality_filter=gcnv_quality_filter, - ) - - await self._assert_expected_search( - [], annotations=NEW_SV_FILTER, quality_filter=gcnv_quality_filter, omit_data_type='SNV_INDEL', - ) - - sv_quality_filter = {'min_gq_sv': 40} - await self._assert_expected_search( - [SV_VARIANT3, SV_VARIANT4], quality_filter=sv_quality_filter, sample_data=SV_WGS_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [], annotations=NEW_SV_FILTER, quality_filter=sv_quality_filter, sample_data=SV_WGS_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [VARIANT2, MULTI_FAMILY_VARIANT], quality_filter={'min_gq': 40, 'vcf_filter': 'pass'}, omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - quality_filter={'min_gq': 60, 'min_qs': 10, 'affected_only': True}, - ) - - await self._assert_expected_search( - [SV_VARIANT3, SV_VARIANT4], quality_filter={'min_gq_sv': 60, 'affected_only': True}, sample_data=SV_WGS_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [VARIANT1, VARIANT2, FAMILY_3_VARIANT], quality_filter={'min_ab': 50}, omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [VARIANT2, VARIANT3], quality_filter={'min_ab': 70, 'affected_only': True}, - omit_data_type='SV_WES', - ) - - quality_filter.update({'min_gq': 40, 'min_ab': 50}) - await self._assert_expected_search( - [VARIANT2, FAMILY_3_VARIANT], quality_filter=quality_filter, omit_data_type='SV_WES', - ) - - annotations = {'splice_ai': '0.0'} # Ensures no variants are filtered out by annotation/path filters - await self._assert_expected_search( - [VARIANT1, VARIANT2, FAMILY_3_VARIANT, MITO_VARIANT1, MITO_VARIANT3], quality_filter=quality_filter, omit_data_type='SV_WES', - annotations=annotations, pathogenicity={'clinvar': ['likely_pathogenic', 'vus_or_conflicting']}, - sample_data={**EXPECTED_SAMPLE_DATA, **FAMILY_2_MITO_SAMPLE_DATA}, - ) - - await self._assert_expected_search( - [VARIANT2, FAMILY_3_VARIANT], quality_filter=quality_filter, omit_data_type='SV_WES', - annotations=annotations, pathogenicity={'clinvar': ['pathogenic']}, - ) - - async def test_location_search(self): - await self._assert_expected_search( - [MULTI_FAMILY_VARIANT, VARIANT4], omit_data_type='SV_WES', **LOCATION_SEARCH, - ) - - # Test "large" gene list search - await self._assert_expected_search( - [VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4], omit_data_type='SV_WES', intervals=LOCATION_SEARCH['intervals'], - gene_ids=LOCATION_SEARCH['gene_ids'] + ['ENSG00000277258', 'ENSG00000275023'], - ) - - await self._assert_expected_search( - [GRCH37_VARIANT], intervals=[['7', 143268894, 143271480]], genome_version='GRCh37', sample_data=FAMILY_2_VARIANT_SAMPLE_DATA) - - sv_intervals = [['1', 9310023, 9380264], ['17', 38717636, 38724781]] - await self._assert_expected_search( - [GCNV_VARIANT3, GCNV_VARIANT4], intervals=sv_intervals, gene_ids=['ENSG00000275023'], omit_data_type='SNV_INDEL', - ) - - await self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT2], sample_data=SV_WGS_SAMPLE_DATA, intervals=sv_intervals, gene_ids=['ENSG00000171621'], - ) - - await self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT2, MULTI_PROJECT_GCNV_VARIANT3, GCNV_VARIANT4], intervals=sv_intervals, - sample_data={'SV_WES': EXPECTED_SAMPLE_DATA['SV_WES'] + SECOND_PROJECT_SV_WES_SAMPLE_DATA, **SV_WGS_SAMPLE_DATA}, - ) - - await self._assert_expected_search( - [VARIANT1, VARIANT2], omit_data_type='SV_WES', **EXCLUDE_LOCATION_SEARCH, - ) - - await self._assert_expected_search( - [GCNV_VARIANT1, GCNV_VARIANT2], intervals=sv_intervals, exclude_intervals=True, omit_data_type='SNV_INDEL', - ) - - await self._assert_expected_search( - [SV_VARIANT3, SV_VARIANT4], sample_data=SV_WGS_SAMPLE_DATA, intervals=sv_intervals, exclude_intervals=True, - ) - - await self._assert_expected_search( - [SELECTED_TRANSCRIPT_MULTI_FAMILY_VARIANT], omit_data_type='SV_WES', - intervals=LOCATION_SEARCH['intervals'][-1:], gene_ids=LOCATION_SEARCH['gene_ids'][1:] - ) - - await self._assert_expected_search( - [GCNV_VARIANT4], padded_interval={'chrom': '17', 'start': 38720781, 'end': 38738703, 'padding': 0.2}, - omit_data_type='SNV_INDEL', - ) - - await self._assert_expected_search( - [], padded_interval={'chrom': '17', 'start': 38720781, 'end': 38738703, 'padding': 0.1}, - omit_data_type='SNV_INDEL', - ) - - await self._assert_expected_search( - [SV_VARIANT4], padded_interval={'chrom': '14', 'start': 106692244, 'end': 106742587, 'padding': 0.1}, - sample_data=SV_WGS_SAMPLE_DATA, - ) - - # For gene search, return SVs annotated in gene even if they fall outside the gene interval - nearest_tss_gene_intervals = [['1', 9292894, 9369532]] - await self._assert_expected_search( - [SV_VARIANT1], sample_data=SV_WGS_SAMPLE_DATA, intervals=nearest_tss_gene_intervals, - ) - await self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT2], sample_data=SV_WGS_SAMPLE_DATA, intervals=nearest_tss_gene_intervals, - gene_ids=['ENSG00000171621'], - ) - - async def test_cluster_intervals(self): - intervals = [ - ['1', 11785723, 11806455], ['1', 91500851, 91525764], ['2', 1234, 5678], ['2', 12345, 67890], - ['7', 1, 11100], ['7', 202020, 20202020], - ] - - self.assertListEqual(BaseHailTableQuery.cluster_intervals(intervals, max_intervals=5), [ - ['1', 11785723, 11806455], ['1', 91500851, 91525764], ['2', 1234, 67890], - ['7', 1, 11100], ['7', 202020, 20202020], - ]) - - self.assertListEqual(BaseHailTableQuery.cluster_intervals(intervals, max_intervals=4), [ - ['1', 11785723, 11806455], ['1', 91500851, 91525764], ['2', 1234, 67890], ['7', 1, 20202020], - ]) - - self.assertListEqual(BaseHailTableQuery.cluster_intervals(intervals, max_intervals=3), [ - ['1', 11785723, 91525764], ['2', 1234, 67890], ['7', 1, 20202020], - ]) - - - async def test_variant_id_search(self): - await self._assert_expected_search([VARIANT2], omit_data_type='SV_WES', **RSID_SEARCH) - - await self._assert_expected_search([VARIANT1], omit_data_type='SV_WES', **VARIANT_ID_SEARCH) - - await self._assert_expected_search( - [VARIANT1], omit_data_type='SV_WES', variant_ids=VARIANT_ID_SEARCH['variant_ids'][:1], - ) - - await self._assert_expected_search( - [], omit_data_type='SV_WES', variant_ids=VARIANT_ID_SEARCH['variant_ids'][1:], - ) - - variant_keys = ['suffix_95340_DUP', 'suffix_140608_DUP'] - await self._assert_expected_search([GCNV_VARIANT1, GCNV_VARIANT4], omit_data_type='SNV_INDEL', variant_keys=variant_keys) - - await self._assert_expected_search([VARIANT1, GCNV_VARIANT1, GCNV_VARIANT4], variant_keys=variant_keys, **VARIANT_ID_SEARCH) - - await self._assert_expected_search([SV_VARIANT2, SV_VARIANT4], sample_data=SV_WGS_SAMPLE_DATA, variant_keys=[ - 'cohort_2911.chr1.final_cleanup_INS_chr1_160', 'phase2_DEL_chr14_4640', - ]) - - async def test_variant_lookup(self): - body = {'genome_version': 'GRCh38', 'variant_id': VARIANT_ID_SEARCH['variant_ids'][0]} - async with self.client.request('POST', '/lookup', json=body) as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertDictEqual(resp_json, VARIANT_LOOKUP_VARIANT) - - body['variant_id'] = VARIANT_ID_SEARCH['variant_ids'][1] - async with self.client.request('POST', '/lookup', json=body) as resp: - self.assertEqual(resp.status, 404) - - body.update({'genome_version': 'GRCh37', 'variant_id': ['7', 143270172, 'A', 'G']}) - async with self.client.request('POST', '/lookup', json=body) as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertDictEqual(resp_json, { - **{k: v for k, v in GRCH37_VARIANT.items() if k not in {'familyGuids', 'genotypes'}}, - 'familyGenotypes': {GRCH37_VARIANT['familyGuids'][0]: [ - {k: v for k, v in g.items() if k != 'individualGuid'} for g in GRCH37_VARIANT['genotypes'].values() - ]}, - }) - - body.update({'variant_id': ['M', 4429, 'G', 'A'], 'data_type': 'MITO', 'genome_version': 'GRCh38'}) - async with self.client.request('POST', '/lookup', json=body) as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertDictEqual(resp_json, { - **{k: v for k, v in MITO_VARIANT1.items() if k not in {'familyGuids', 'genotypes'}}, - 'familyGenotypes': {MITO_VARIANT1['familyGuids'][0]: [ - {k: v for k, v in g.items() if k != 'individualGuid'} for g in MITO_VARIANT1['genotypes'].values() - ]}, - }) - - body.update({'variant_id': 'phase2_DEL_chr14_4640', 'data_type': 'SV_WGS', 'sample_data': SV_WGS_SAMPLE_DATA['SV_WGS']}) - async with self.client.request('POST', '/lookup', json=body) as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertDictEqual(resp_json, SV_VARIANT4) - - body.update({'variant_id': 'suffix_140608_DUP', 'data_type': 'SV_WES', 'sample_data': EXPECTED_SAMPLE_DATA['SV_WES']}) - async with self.client.request('POST', '/lookup', json=body) as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertDictEqual(resp_json, { - **NO_GENOTYPE_GCNV_VARIANT, 'genotypes': { - individual: {k: v for k, v in genotype.items() if k not in {'start', 'end', 'numExon', 'geneIds'}} - for individual, genotype in GCNV_VARIANT4['genotypes'].items() - } - }) - - body['variant_id'] = 'suffix_140608_DEL' - async with self.client.request('POST', '/lookup', json=body) as resp: - self.assertEqual(resp.status, 404) - - async def test_multi_variant_lookup(self): - await self._test_multi_lookup(VARIANT_ID_SEARCH['variant_ids'], 'SNV_INDEL', [VARIANT1]) - - await self._test_multi_lookup([['7', 143270172, 'A', 'G']], 'SNV_INDEL', [GRCH37_VARIANT], genome_version='GRCh37') - - await self._test_multi_lookup([['M', 4429, 'G', 'A'], ['M', 14783, 'T', 'C']], 'MITO', [MITO_VARIANT1, MITO_VARIANT3]) - - await self._test_multi_lookup( - ['cohort_2911.chr1.final_cleanup_INS_chr1_160', 'phase2_DEL_chr14_4640'], - 'SV_WGS', [SV_VARIANT2, SV_VARIANT4], - ) - - await self._test_multi_lookup(['suffix_140608_DUP'], 'SV_WES', [NO_GENOTYPE_GCNV_VARIANT]) - - async def _test_multi_lookup(self, variant_ids, data_type, results, genome_version='GRCh38'): - body = {'genome_version': genome_version, 'data_type': data_type, 'variant_ids': variant_ids} - async with self.client.request('POST', '/multi_lookup', json=body) as resp: - self.assertEqual(resp.status, 200) - resp_json = await resp.json() - self.assertDictEqual(resp_json, {'results': [ - {k: v for k, v in variant.items() if k not in {'familyGuids', 'genotypes'}} - for variant in results - ]}) - - async def test_frequency_filter(self): - sv_callset_filter = {'sv_callset': {'af': 0.05}} - # seqr af filter is ignored for SNV_INDEL - await self._assert_expected_search( - [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - frequencies={'seqr': {'af': 0.2}, **sv_callset_filter}, - ) - - await self._assert_expected_search( - [MULTI_FAMILY_VARIANT, VARIANT4, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - frequencies={'seqr': {'ac': 5}, **sv_callset_filter}, - ) - - await self._assert_expected_search( - [MULTI_FAMILY_VARIANT, VARIANT4], frequencies={'seqr': {'ac': 4}}, omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [MULTI_FAMILY_VARIANT, VARIANT4], frequencies={'seqr': {'hh': 1}}, omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [VARIANT4], frequencies={'seqr': {'ac': 4, 'hh': 0}}, omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [MITO_VARIANT1, MITO_VARIANT2], frequencies={'seqr': {'af': 0.01}}, sample_data=FAMILY_2_MITO_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [SV_VARIANT1], frequencies=sv_callset_filter, sample_data=SV_WGS_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [VARIANT1, VARIANT2, VARIANT4], frequencies={'gnomad_genomes': {'af': 0.05}}, omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [VARIANT2, VARIANT4], frequencies={'gnomad_genomes': {'af': 0.05, 'hh': 1}}, omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [VARIANT2, VARIANT4, MITO_VARIANT1, MITO_VARIANT2], sample_data=FAMILY_2_ALL_SAMPLE_DATA, - frequencies={'gnomad_genomes': {'af': 0.005}, 'gnomad_mito': {'af': 0.05}}, - ) - - await self._assert_expected_search( - [SV_VARIANT1, SV_VARIANT3, SV_VARIANT4], frequencies={'gnomad_svs': {'af': 0.001}}, sample_data=SV_WGS_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [VARIANT4], frequencies={'seqr': {'ac': 10}, 'gnomad_genomes': {'ac': 50}}, - omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4], frequencies={'seqr': {}, 'gnomad_genomes': {'af': None}}, - omit_data_type='SV_WES', - ) - - annotations = {'splice_ai': '0.0'} # Ensures no variants are filtered out by annotation/path filters - await self._assert_expected_search( - [VARIANT1, VARIANT2, VARIANT4], frequencies={'gnomad_genomes': {'af': 0.01, 'hh': 10}}, omit_data_type='SV_WES', - annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'likely_pathogenic', 'vus_or_conflicting']}, - ) - - await self._assert_expected_search( - [VARIANT2, VARIANT4], frequencies={'gnomad_genomes': {'af': 0.01}}, omit_data_type='SV_WES', - annotations=annotations, pathogenicity={'clinvar': ['pathogenic', 'vus_or_conflicting']}, - ) - - async def test_annotations_filter(self): - await self._assert_expected_search([VARIANT2], pathogenicity={'hgmd': ['hgmd_other']}, omit_data_type='SV_WES') - - pathogenicity = {'clinvar': ['likely_pathogenic', 'vus_or_conflicting', 'benign']} - await self._assert_expected_search( - [VARIANT1, VARIANT2, MITO_VARIANT1, MITO_VARIANT3], pathogenicity=pathogenicity, sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - exclude = {'clinvar': pathogenicity['clinvar'][1:]} - pathogenicity['clinvar'] = pathogenicity['clinvar'][:1] - annotations = {'SCREEN': ['CTCF-only', 'DNase-only'], 'UTRAnnotator': ['5_prime_UTR_stop_codon_loss_variant']} - selected_transcript_variant_2 = {**VARIANT2, 'selectedMainTranscriptId': 'ENST00000408919'} - await self._assert_expected_search( - [VARIANT1, selected_transcript_variant_2, VARIANT4, MITO_VARIANT3], pathogenicity=pathogenicity, annotations=annotations, - sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [VARIANT1, VARIANT4, MITO_VARIANT3], exclude=exclude, pathogenicity=pathogenicity, - annotations=annotations, sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [], pathogenicity=pathogenicity, annotations=annotations, sample_data=FAMILY_2_VARIANT_SAMPLE_DATA, - genome_version='GRCh37', - ) - - annotations = { - 'missense': ['missense_variant'], 'in_frame': ['inframe_insertion', 'inframe_deletion'], 'frameshift': None, - 'structural_consequence': ['INTRONIC', 'LOF'], - } - await self._assert_expected_search( - [VARIANT1, VARIANT2, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4, MITO_VARIANT2, MITO_VARIANT3], pathogenicity=pathogenicity, - annotations=annotations, sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [VARIANT2, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4, GCNV_VARIANT3, GCNV_VARIANT4], annotations=annotations, - ) - - await self._assert_expected_search([SV_VARIANT1], annotations=annotations, sample_data=SV_WGS_SAMPLE_DATA) - - annotations['splice_ai'] = '0.005' - annotations['structural'] = ['gCNV_DUP', 'DEL'] - await self._assert_expected_search( - [VARIANT2, MULTI_FAMILY_VARIANT, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - annotations=annotations, - ) - - await self._assert_expected_search([SV_VARIANT1, SV_VARIANT4], annotations=annotations, sample_data=SV_WGS_SAMPLE_DATA) - - annotations = {'other': ['non_coding_transcript_exon_variant__canonical', 'non_coding_transcript_exon_variant']} - await self._assert_expected_search( - [VARIANT1, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_3, MITO_VARIANT1, MITO_VARIANT3], - pathogenicity=pathogenicity, annotations=annotations, sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2], - gene_ids=LOCATION_SEARCH['gene_ids'][1:], annotations=annotations, omit_data_type='SV_WES', - ) - - annotations['other'].append('intron_variant') - await self._assert_expected_search( - [SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2, SELECTED_TRANSCRIPT_MULTI_FAMILY_VARIANT], - gene_ids=LOCATION_SEARCH['gene_ids'][1:], annotations=annotations, omit_data_type='SV_WES', - ) - - annotations['other'] = annotations['other'][:1] - annotations['splice_ai'] = '0.005' - await self._assert_expected_search( - [VARIANT1, VARIANT3, MITO_VARIANT1, MITO_VARIANT3], - pathogenicity=pathogenicity, annotations=annotations, sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - annotations['extended_splice_site'] = ['extended_intronic_splice_region_variant'] - await self._assert_expected_search( - [VARIANT1, VARIANT3, VARIANT4, MITO_VARIANT1, MITO_VARIANT3], - pathogenicity=pathogenicity, annotations=annotations, sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - annotations = {'motif_feature': ['TF_binding_site_variant'], 'regulatory_feature': ['regulatory_region_variant']} - await self._assert_expected_search( - [VARIANT3, VARIANT4], annotations=annotations, sample_data=FAMILY_2_VARIANT_SAMPLE_DATA, - ) - - async def test_secondary_annotations_filter(self): - annotations_1 = {'missense': ['missense_variant']} - annotations_2 = {'other': ['intron_variant']} - - await self._assert_expected_search( - [[VARIANT3, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4]], inheritance_mode='compound_het', omit_data_type='SV_WES', - annotations=annotations_1, annotations_secondary=annotations_2, - ) - - await self._assert_expected_search( - [VARIANT2, [VARIANT3, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4]], inheritance_mode='recessive', omit_data_type='SV_WES', - annotations=annotations_1, annotations_secondary=annotations_2, - ) - - await self._assert_expected_search( - [[VARIANT3, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4]], inheritance_mode='recessive', omit_data_type='SV_WES', - annotations=annotations_2, annotations_secondary=annotations_1, - ) - - gcnv_annotations_1 = {'structural': ['gCNV_DUP']} - gcnv_annotations_2 = {'structural_consequence': ['LOF'], 'structural': []} - - await self._assert_expected_search( - [[GCNV_VARIANT3, GCNV_VARIANT4]], omit_data_type='SNV_INDEL', inheritance_mode='compound_het', - annotations=gcnv_annotations_1, annotations_secondary=gcnv_annotations_2, - ) - - await self._assert_expected_search( - [GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], omit_data_type='SNV_INDEL', inheritance_mode='recessive', - annotations=gcnv_annotations_2, annotations_secondary=gcnv_annotations_1, - ) - - # Do not return pairs where annotations match in a non-paired gene - gcnv_annotations_no_pair = {'structural_consequence': ['COPY_GAIN']} - await self._assert_expected_search( - [], omit_data_type='SNV_INDEL', inheritance_mode='compound_het', - annotations=gcnv_annotations_1, annotations_secondary=gcnv_annotations_no_pair, - ) - - await self._assert_expected_search( - [], omit_data_type='SNV_INDEL', inheritance_mode='compound_het', - annotations={**gcnv_annotations_1, **gcnv_annotations_no_pair}, - ) - - await self._assert_expected_search( - [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4]], inheritance_mode='compound_het', - annotations=annotations_1, annotations_secondary=gcnv_annotations_2, - ) - - await self._assert_expected_search( - [VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4], GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], - inheritance_mode='recessive', - annotations={**annotations_1, **gcnv_annotations_1}, annotations_secondary={**annotations_2, **gcnv_annotations_2}, - ) - - await self._assert_expected_search( - [VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], MULTI_PROJECT_GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], - inheritance_mode='recessive', - annotations={**annotations_1, **gcnv_annotations_1}, annotations_secondary={**annotations_2, **gcnv_annotations_2}, - intervals=[['1', 38717636, 38724781], ['17', 38717636, 38724781]], - sample_data={**EXPECTED_SAMPLE_DATA, 'SV_WES': EXPECTED_SAMPLE_DATA['SV_WES'] + SECOND_PROJECT_SV_WES_SAMPLE_DATA}, - ) - - await self._assert_expected_search( - [MULTI_DATA_TYPE_COMP_HET_VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [GCNV_VARIANT3, GCNV_VARIANT4]], - inheritance_mode='recessive', - annotations={**annotations_1, 'structural': ['gCNV_DEL'], 'structural_consequence': ['INTRONIC']}, - annotations_secondary={**annotations_2, **gcnv_annotations_1}, - gene_ids=['ENSG00000277258', 'ENSG00000275023'], intervals=[['1', 38717636, 38724781], ['17', 38717636, 38724781]], - ) - - await self._assert_expected_search( - [MULTI_DATA_TYPE_COMP_HET_VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4]], - inheritance_mode='recessive', annotations={**annotations_1, 'structural': [], 'structural_consequence': []}, - annotations_secondary=gcnv_annotations_2, - gene_ids=['ENSG00000277258', 'ENSG00000275023'], intervals=[['1', 38717636, 38724781], ['17', 38717636, 38724781]], - ) - - sv_annotations_1 = {'structural': ['INS', 'LOF']} - sv_annotations_2 = {'structural': ['DEL', 'gCNV_DUP'], 'structural_consequence': ['INTRONIC']} - - await self._assert_expected_search( - [[SV_VARIANT1, SV_VARIANT2]], sample_data=SV_WGS_SAMPLE_DATA, inheritance_mode='compound_het', - annotations=sv_annotations_1, annotations_secondary=sv_annotations_2, - ) - - await self._assert_expected_search( - [[SV_VARIANT1, SV_VARIANT2], SV_VARIANT4], sample_data=SV_WGS_SAMPLE_DATA, inheritance_mode='recessive', - annotations=sv_annotations_2, annotations_secondary=sv_annotations_1, - ) - - pathogenicity = {'clinvar': ['likely_pathogenic', 'vus_or_conflicting']} - await self._assert_expected_search( - [VARIANT2, [VARIANT3, SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_4]], inheritance_mode='recessive', omit_data_type='SV_WES', - annotations=annotations_2, annotations_secondary=annotations_1, pathogenicity=pathogenicity, - ) - - await self._assert_expected_search( - [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [GCNV_VARIANT3, GCNV_VARIANT4]], - inheritance_mode='compound_het', pathogenicity=pathogenicity, - annotations=gcnv_annotations_2, annotations_secondary=gcnv_annotations_1, - ) - - await self._assert_expected_search( - [VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], - inheritance_mode='recessive', pathogenicity=pathogenicity, - annotations=gcnv_annotations_2, annotations_secondary=gcnv_annotations_1, - ) - - selected_transcript_annotations = {'other': ['non_coding_transcript_exon_variant']} - await self._assert_expected_search( - [VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], GCNV_VARIANT3], - inheritance_mode='recessive', pathogenicity=pathogenicity, - annotations=gcnv_annotations_2, annotations_secondary=selected_transcript_annotations, - ) - - # Search works with a different number of samples within the family - missing_gt_gcnv_variant = { - **GCNV_VARIANT4, 'genotypes': {k: v for k, v in GCNV_VARIANT4['genotypes'].items() if k != 'I000005_hg00732'} - } - await self._assert_expected_search( - [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, missing_gt_gcnv_variant]], - inheritance_mode='compound_het', pathogenicity=pathogenicity, - annotations=gcnv_annotations_2, annotations_secondary=selected_transcript_annotations, - sample_data={**EXPECTED_SAMPLE_DATA, 'SV_WES': SV_WES_SAMPLE_DATA['SV_WES'][:1] + SV_WES_SAMPLE_DATA['SV_WES'][2:]} - - ) - - # Do not return pairs where annotations match in a non-paired gene - await self._assert_expected_search( - [GCNV_VARIANT3], inheritance_mode='recessive', - annotations=gcnv_annotations_2, annotations_secondary=selected_transcript_annotations, - ) - - screen_annotations = {'SCREEN': ['CTCF-only']} - await self._assert_expected_search( - [], inheritance_mode='recessive', omit_data_type='SV_WES', - annotations=screen_annotations, annotations_secondary=annotations_1, - ) - - await self._assert_expected_search( - [[VARIANT3, VARIANT4]], inheritance_mode='recessive', omit_data_type='SV_WES', - annotations=screen_annotations, annotations_secondary=annotations_2, - ) - - await self._assert_expected_search( - [VARIANT2, [SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_3, VARIANT4]], inheritance_mode='recessive', - annotations=screen_annotations, annotations_secondary=selected_transcript_annotations, - pathogenicity=pathogenicity, omit_data_type='SV_WES', - ) - - await self._assert_expected_search( - [SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2, [VARIANT3, VARIANT4]], - annotations={**selected_transcript_annotations, **screen_annotations}, annotations_secondary=annotations_2, - inheritance_mode='recessive', omit_data_type='SV_WES', - ) - - async def test_in_silico_filter(self): - in_silico = {'eigen': '3.5', 'mut_taster': 'N', 'vest': 0.5} - await self._assert_expected_search( - [VARIANT1, VARIANT4, MITO_VARIANT1, MITO_VARIANT2, MITO_VARIANT3], in_silico=in_silico, - sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [GRCH37_VARIANT], genome_version='GRCh37', in_silico=in_silico, - sample_data=FAMILY_2_VARIANT_SAMPLE_DATA, - ) - - in_silico['requireScore'] = True - in_silico.pop('eigen') - await self._assert_expected_search( - [VARIANT4, MITO_VARIANT2], in_silico=in_silico, sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - sv_in_silico = {'strvctvre': 0.1, 'requireScore': True} - await self._assert_expected_search( - [GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], omit_data_type='SNV_INDEL', in_silico=sv_in_silico, - ) - - await self._assert_expected_search( - [SV_VARIANT4], sample_data=SV_WGS_SAMPLE_DATA, in_silico=sv_in_silico, - ) - - async def test_search_errors(self): - search_body = get_hail_search_body(sample_data=FAMILY_2_MISSING_SAMPLE_DATA) - async with self.client.request('POST', '/search', json=search_body) as resp: - self.assertEqual(resp.status, 400) - reason = resp.reason - self.assertEqual(reason, 'The following samples are available in seqr but missing the loaded data: NA19675_1, NA19678') - - search_body = get_hail_search_body(sample_data=MULTI_PROJECT_MISSING_SAMPLE_DATA) - async with self.client.request('POST', '/search', json=search_body) as resp: - self.assertEqual(resp.status, 400) - reason = resp.reason - self.assertEqual(reason, 'The following samples are available in seqr but missing the loaded data: NA19675_1, NA19678') - - search_body = get_hail_search_body( - intervals=LOCATION_SEARCH['intervals'] + [['1', 1, 999999999]], omit_data_type='SV_WES', - ) - async with self.client.request('POST', '/search', json=search_body) as resp: - self.assertEqual(resp.status, 400) - reason = resp.reason - self.assertEqual(reason, 'Invalid intervals: 1:1-999999999') - - async def test_sort(self): - await self._assert_expected_search( - [_sorted(VARIANT4, [2, 2]), _sorted(MITO_VARIANT2, [11, 11]), _sorted(VARIANT2, [12, 12]), - _sorted(MITO_VARIANT3, [17, 17]), _sorted(MITO_VARIANT1, [22, 22]), _sorted(VARIANT3, [26, 27]), - _sorted(VARIANT1, [None, None])], sample_data=FAMILY_2_ALL_SAMPLE_DATA, sort='protein_consequence', - ) - - await self._assert_expected_search( - [_sorted(GCNV_VARIANT2, [0]), _sorted(GCNV_VARIANT3, [0]), _sorted(GCNV_VARIANT4, [0]), - _sorted(GCNV_VARIANT1, [3])], omit_data_type='SNV_INDEL', sort='protein_consequence', - ) - - await self._assert_expected_search( - [_sorted(VARIANT4, [2, 2]), _sorted(GCNV_VARIANT2, [4.5, 0]), _sorted(GCNV_VARIANT3, [4.5, 0]), _sorted(GCNV_VARIANT4, [4.5, 0]), - _sorted(GCNV_VARIANT1, [4.5, 3]), _sorted(VARIANT2, [12, 12]), - _sorted(MULTI_FAMILY_VARIANT, [26, 27]), _sorted(VARIANT1, [None, None])], sort='protein_consequence', - ) - - await self._assert_expected_search( - [_sorted(SV_VARIANT1, [11]), _sorted(SV_VARIANT2, [12]), _sorted(SV_VARIANT3, [12]), _sorted(SV_VARIANT4, [12])], - sample_data=SV_WGS_SAMPLE_DATA, sort='protein_consequence', - ) - - await self._assert_expected_search( - [_sorted(VARIANT4, [2, 2]), _sorted(SELECTED_ANNOTATION_TRANSCRIPT_VARIANT_2, [12, 26]), - _sorted(SELECTED_ANNOTATION_TRANSCRIPT_MULTI_FAMILY_VARIANT, [26, 26])], - omit_data_type='SV_WES', sort='protein_consequence', - annotations={'other': ['non_coding_transcript_exon_variant'], 'splice_ai': '0'}, - ) - - await self._assert_expected_search( - [_sorted(VARIANT1, [4]), _sorted(VARIANT2, [8]), _sorted(MULTI_FAMILY_VARIANT, [12.5]), - _sorted(VARIANT4, [12.5]), GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], sort='pathogenicity', - ) - - await self._assert_expected_search( - [ _sorted(MITO_VARIANT3, [4]), _sorted(VARIANT1, [4, None]), _sorted(VARIANT2, [8, 3]), - _sorted(MITO_VARIANT1, [11]), _sorted(MITO_VARIANT2, [12.5]), _sorted(VARIANT3, [12.5, None]), - _sorted(VARIANT4, [12.5, None])], - sort='pathogenicity_hgmd', sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [_sorted(VARIANT2, [0]), _sorted(MITO_VARIANT1, [0]), _sorted(MITO_VARIANT2, [0]), - _sorted(VARIANT4, [0.00026519427774474025]), _sorted(VARIANT1, [0.034449315071105957]), - _sorted(MITO_VARIANT3, [0.05534649267792702]), _sorted(VARIANT3, [0.38041073083877563])], - sort='gnomad', sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [_sorted(VARIANT1, [0]), _sorted(MULTI_FAMILY_VARIANT, [0]), _sorted(VARIANT4, [0]), - _sorted(VARIANT2, [0.28899794816970825]), GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], - sort='gnomad_exomes', - ) - - await self._assert_expected_search( - [_sorted(VARIANT4, [2]),_sorted(MULTI_FAMILY_VARIANT, [4]), _sorted(VARIANT1, [9]), - _sorted(VARIANT2, [28]), _sorted(GCNV_VARIANT3, [35]), _sorted(GCNV_VARIANT4, [115]), - _sorted(GCNV_VARIANT2, [284]), _sorted(GCNV_VARIANT1, [1763])], - sort='callset_af', - ) - - await self._assert_expected_search( - [_sorted(MITO_VARIANT1, [0]), _sorted(MITO_VARIANT2, [0]), _sorted(VARIANT4, [2]), - _sorted(MITO_VARIANT3, [3]), _sorted(VARIANT3, [4]), _sorted(VARIANT1, [9]), - _sorted(VARIANT2, [28])], - sort='callset_af', sample_data=FAMILY_2_ALL_SAMPLE_DATA, - ) - - await self._assert_expected_search( - [_sorted(VARIANT4, [-29.899999618530273]), _sorted(VARIANT2, [-20.899999618530273]), - _sorted(VARIANT1, [-4.668000221252441]), _sorted(MULTI_FAMILY_VARIANT, [-2.753999948501587]), - GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], sort='cadd', - ) - - await self._assert_expected_search( - [_sorted(VARIANT4, [-0.5260000228881836]), _sorted(VARIANT2, [-0.19699999690055847]), - _sorted(VARIANT1, [0]), _sorted(MULTI_FAMILY_VARIANT, [0])], omit_data_type='SV_WES', sort='revel', - ) - - await self._assert_expected_search( - [_sorted(MULTI_FAMILY_VARIANT, [-0.009999999776482582]), _sorted(VARIANT2, [0]), _sorted(VARIANT4, [0]), - _sorted(VARIANT1, [0])], omit_data_type='SV_WES', sort='splice_ai', - ) - - await self._assert_expected_search( - [_sorted(VARIANT2, [-0.9977999925613403, -0.9977999925613403]), _sorted(VARIANT1, [0, 0]), - _sorted(MULTI_FAMILY_VARIANT, [0, 0]), _sorted(VARIANT4, [0, 0])], omit_data_type='SV_WES', sort='alphamissense', - ) - - sort = 'in_omim' - await self._assert_expected_search( - [_sorted(MULTI_FAMILY_VARIANT, [0, -2]), _sorted(VARIANT2, [0, -1]), _sorted(VARIANT4, [0, -1]), _sorted(VARIANT1, [1, 0])], - omit_data_type='SV_WES', sort=sort, sort_metadata=OMIM_SORT_METADATA, - ) - - await self._assert_expected_search( - [_sorted(GCNV_VARIANT3, [-1]), _sorted(GCNV_VARIANT4, [-1]), _sorted(GCNV_VARIANT1, [0]), _sorted(GCNV_VARIANT2, [0])], - omit_data_type='SNV_INDEL', sort=sort, sort_metadata=OMIM_SORT_METADATA, - ) - - await self._assert_expected_search( - [_sorted(MULTI_FAMILY_VARIANT, [0, -2]), _sorted(VARIANT2, [0, -1]), _sorted(VARIANT4, [0, -1]), - _sorted(GCNV_VARIANT3, [0, -1]), _sorted(GCNV_VARIANT4, [0, -1]), _sorted(GCNV_VARIANT1, [0, 0]), - _sorted(GCNV_VARIANT2, [0, 0]), _sorted(VARIANT1, [1, 0])], sort=sort, sort_metadata=OMIM_SORT_METADATA, - ) - - await self._assert_expected_search( - [_sorted(VARIANT2, [0, -1]), _sorted(MULTI_FAMILY_VARIANT, [1, -1]), _sorted(VARIANT1, [1, 0]), _sorted(VARIANT4, [1, 0])], - omit_data_type='SV_WES', sort=sort, sort_metadata=['ENSG00000177000'], - ) - - constraint_sort_metadata = {'ENSG00000177000': 2, 'ENSG00000275023': 3, 'ENSG00000097046': 4} - sort = 'constraint' - await self._assert_expected_search( - [_sorted(VARIANT2, [2, 2]), _sorted(MULTI_FAMILY_VARIANT, [4, 2]), _sorted(VARIANT4, [4, 4]), - _sorted(VARIANT1, [None, None])], omit_data_type='SV_WES', sort=sort, sort_metadata=constraint_sort_metadata, - ) - - await self._assert_expected_search( - [_sorted(GCNV_VARIANT3, [3]), _sorted(GCNV_VARIANT4, [3]), _sorted(GCNV_VARIANT1, [None]), - _sorted(GCNV_VARIANT2, [None])], omit_data_type='SNV_INDEL', sort=sort, sort_metadata=constraint_sort_metadata, - ) - - await self._assert_expected_search( - [_sorted(VARIANT2, [2, 2]), _sorted(GCNV_VARIANT3, [3, 3]), _sorted(GCNV_VARIANT4, [3, 3]), - _sorted(MULTI_FAMILY_VARIANT, [4, 2]), _sorted(VARIANT4, [4, 4]), _sorted(VARIANT1, [None, None]), - _sorted(GCNV_VARIANT1, [None, None]), _sorted(GCNV_VARIANT2, [None, None])], - sort=sort, sort_metadata=constraint_sort_metadata, - ) - - await self._assert_expected_search( - [_sorted(VARIANT2, [3, 3]), _sorted(MULTI_FAMILY_VARIANT, [None, 3]), _sorted(VARIANT1, [None, None]), - _sorted(VARIANT4, [None, None])], omit_data_type='SV_WES', sort='prioritized_gene', - sort_metadata={'ENSG00000177000': 3}, - ) - - await self._assert_expected_search( - [_sorted(MULTI_PROJECT_VARIANT1, [2]), _sorted(MULTI_PROJECT_VARIANT2, [2]), - _sorted(VARIANT3, [2]), _sorted(VARIANT4, [2]), _sorted(PROJECT_2_VARIANT, [11])], - sort='family_guid', sample_data=MULTI_PROJECT_SAMPLE_DATA, - ) - - # size sort only applies to SVs, so has no impact on other variant - await self._assert_expected_search( - [_sorted(GCNV_VARIANT1, [-171766]), _sorted(GCNV_VARIANT2, [-17768]), _sorted(GCNV_VARIANT4, [-14487]), - _sorted(GCNV_VARIANT3, [-2666]), VARIANT1, VARIANT2, MULTI_FAMILY_VARIANT, VARIANT4], sort='size', - ) - - await self._assert_expected_search( - [_sorted(SV_VARIANT4, [-46343]), _sorted(SV_VARIANT1, [-104]), _sorted(SV_VARIANT2, [-50]), - _sorted(SV_VARIANT3, [-50])], sample_data=SV_WGS_SAMPLE_DATA, sort='size', - ) - - # sort applies to compound hets - await self._assert_expected_search( - [[_sorted(VARIANT4, [-0.5260000228881836]), _sorted(VARIANT3, [0])], - _sorted(VARIANT2, [-0.19699999690055847])], - sort='revel', inheritance_mode='recessive', omit_data_type='SV_WES', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(VARIANT3, [-0.009999999776482582]), _sorted(VARIANT4, [0])], _sorted(VARIANT2, [0])], - sort='splice_ai', inheritance_mode='recessive', omit_data_type='SV_WES', **COMP_HET_ALL_PASS_FILTERS, - ) - - async def test_multi_data_type_comp_het_sort(self): - await self._assert_expected_search( - [[_sorted(VARIANT4, [2, 2]), _sorted(VARIANT3, [26, 27])], - _sorted(GCNV_VARIANT3, [4.5, 0]), [_sorted(GCNV_VARIANT3, [0]), _sorted(GCNV_VARIANT4, [0])], - [_sorted(GCNV_VARIANT4, [4.5, 0]), _sorted(MULTI_DATA_TYPE_COMP_HET_VARIANT2, [12, 12])], - _sorted(VARIANT2, [12, 12])], - sort='protein_consequence', inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(GCNV_VARIANT4, [-14487]), _sorted(GCNV_VARIANT3, [-2666])], - [_sorted(GCNV_VARIANT4, [-14487]), MULTI_DATA_TYPE_COMP_HET_VARIANT2], - [VARIANT3, VARIANT4]], - sort='size', inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(MULTI_DATA_TYPE_COMP_HET_VARIANT2, [8]), GCNV_VARIANT4], - [_sorted(VARIANT3, [12.5]), _sorted(VARIANT4, [12.5])], - [GCNV_VARIANT3, GCNV_VARIANT4]], - sort='pathogenicity', inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(VARIANT4, [-0.6869999766349792]), _sorted(VARIANT3, [0])], _sorted(VARIANT2, [0]), - [_sorted(MULTI_DATA_TYPE_COMP_HET_VARIANT2, [0]), GCNV_VARIANT4], - GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], - sort='mut_pred', inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(VARIANT3, [-0.009999999776482582]), _sorted(VARIANT4, [0])], - [_sorted(MULTI_DATA_TYPE_COMP_HET_VARIANT2, [0]), GCNV_VARIANT4], - [GCNV_VARIANT3, GCNV_VARIANT4]], - sort='splice_ai', inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(GCNV_VARIANT3, [-0.7860000133514404]), _sorted(GCNV_VARIANT4, [-0.7099999785423279])], - [_sorted(GCNV_VARIANT4, [-0.7099999785423279]), MULTI_DATA_TYPE_COMP_HET_VARIANT2], - [VARIANT3, VARIANT4]], - sort='strvctvre', inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(VARIANT4, [2]), _sorted(VARIANT3, [4])], - [_sorted(MULTI_DATA_TYPE_COMP_HET_VARIANT2, [28]), _sorted(GCNV_VARIANT4, [115])], - [_sorted(GCNV_VARIANT3, [35]), _sorted(GCNV_VARIANT4, [115])]], - sort='callset_af', inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(VARIANT3, [0]), _sorted(VARIANT4, [0])], - [_sorted(MULTI_DATA_TYPE_COMP_HET_VARIANT2, [0.28899794816970825]), GCNV_VARIANT4], - [GCNV_VARIANT3, GCNV_VARIANT4]], - sort='gnomad_exomes', inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, - ) - - await self._assert_expected_search( - [[_sorted(VARIANT3, [0, -2]), _sorted(VARIANT4, [0, -1])], - [_sorted(GCNV_VARIANT3, [-1]), _sorted(GCNV_VARIANT4, [-1])], - [_sorted(GCNV_VARIANT4, [0, -1]), _sorted(MULTI_DATA_TYPE_COMP_HET_VARIANT2, [1, -1])]], - sort='in_omim', sort_metadata=OMIM_SORT_METADATA, inheritance_mode='compound_het', **COMP_HET_ALL_PASS_FILTERS, - ) diff --git a/hail_search/test_utils.py b/hail_search/test_utils.py deleted file mode 100644 index 6fc725dcac..0000000000 --- a/hail_search/test_utils.py +++ /dev/null @@ -1,1177 +0,0 @@ -from copy import deepcopy - - -FAMILY_3_SAMPLE = { - 'sample_id': 'NA20870', 'individual_guid': 'I000007_na20870', 'family_guid': 'F000003_3', - 'project_guid': 'R0001_1kg', 'affected': 'A', 'sample_type': 'WES', -} -FAMILY_2_VARIANT_SAMPLE_DATA_WITH_SEX = {'SNV_INDEL': [ - {'sample_id': 'HG00731', 'individual_guid': 'I000004_hg00731', 'family_guid': 'F000002_2', 'project_guid': 'R0001_1kg', 'affected': 'A', 'sample_type': 'WES', 'is_male': False}, - {'sample_id': 'HG00732', 'individual_guid': 'I000005_hg00732', 'family_guid': 'F000002_2', 'project_guid': 'R0001_1kg', 'affected': 'N', 'sample_type': 'WES', 'is_male': True}, - {'sample_id': 'HG00733', 'individual_guid': 'I000006_hg00733', 'family_guid': 'F000002_2', 'project_guid': 'R0001_1kg', 'affected': 'N', 'sample_type': 'WES', 'is_male': False}, -]} -FAMILY_2_VARIANT_SAMPLE_DATA = deepcopy(FAMILY_2_VARIANT_SAMPLE_DATA_WITH_SEX) -for s in FAMILY_2_VARIANT_SAMPLE_DATA['SNV_INDEL']: - s.pop('is_male') - -EXPECTED_SAMPLE_DATA_WITH_SEX = { - 'SV_WES': [ - {'sample_id': 'HG00731', 'individual_guid': 'I000004_hg00731', 'family_guid': 'F000002_2', 'project_guid': 'R0001_1kg', 'affected': 'A', 'sample_type': 'WES', 'is_male': False}, - {'sample_id': 'HG00732', 'individual_guid': 'I000005_hg00732', 'family_guid': 'F000002_2', 'project_guid': 'R0001_1kg', 'affected': 'N', 'sample_type': 'WES', 'is_male': True}, - {'sample_id': 'HG00733', 'individual_guid': 'I000006_hg00733', 'family_guid': 'F000002_2', 'project_guid': 'R0001_1kg', 'affected': 'N', 'sample_type': 'WES', 'is_male': False} - ], -} -EXPECTED_SAMPLE_DATA_WITH_SEX.update(FAMILY_2_VARIANT_SAMPLE_DATA_WITH_SEX) -EXPECTED_SAMPLE_DATA_WITH_SEX['SNV_INDEL'].append({'is_male': True, **FAMILY_3_SAMPLE}) - -EXPECTED_SAMPLE_DATA = deepcopy(EXPECTED_SAMPLE_DATA_WITH_SEX) -for samples in EXPECTED_SAMPLE_DATA.values(): - for s in samples: - s.pop('is_male') - -CUSTOM_AFFECTED_SAMPLE_DATA = {'SNV_INDEL': deepcopy(EXPECTED_SAMPLE_DATA['SNV_INDEL'])} -CUSTOM_AFFECTED_SAMPLE_DATA['SNV_INDEL'][0]['affected'] = 'N' -CUSTOM_AFFECTED_SAMPLE_DATA['SNV_INDEL'][1]['affected'] = 'A' -CUSTOM_AFFECTED_SAMPLE_DATA['SNV_INDEL'][2]['affected'] = 'U' - -FAMILY_1_SAMPLE_DATA = { - 'SNV_INDEL': [ - {'sample_id': 'NA19675_1', 'individual_guid': 'I000001_na19675', 'family_guid': 'F000001_1', 'project_guid': 'R0001_1kg', 'sample_type': 'WES', 'affected': 'A'}, - {'sample_id': 'NA19678', 'individual_guid': 'I000002_na19678', 'family_guid': 'F000001_1', 'project_guid': 'R0001_1kg', 'sample_type': 'WES', 'affected': 'N'}, - ], -} -FAMILY_2_MISSING_SAMPLE_DATA = deepcopy(FAMILY_1_SAMPLE_DATA) -for s in FAMILY_2_MISSING_SAMPLE_DATA['SNV_INDEL']: - s['family_guid'] = 'F000002_2' - -FAMILY_2_MITO_SAMPLE_DATA = {'MITO': [ - {'sample_id': 'HG00733', 'individual_guid': 'I000006_hg00733', 'family_guid': 'F000002_2', 'project_guid': 'R0001_1kg', 'affected': 'N', 'sample_type': 'WES'}, -]} -FAMILY_2_ALL_SAMPLE_DATA = deepcopy(FAMILY_2_VARIANT_SAMPLE_DATA) -FAMILY_2_ALL_SAMPLE_DATA.update(FAMILY_2_MITO_SAMPLE_DATA) - -ALL_AFFECTED_SAMPLE_DATA = deepcopy(EXPECTED_SAMPLE_DATA) -ALL_AFFECTED_SAMPLE_DATA.update(FAMILY_2_MITO_SAMPLE_DATA) -FAMILY_5_SAMPLE = { - 'sample_id': 'NA20874', 'individual_guid': 'I000009_na20874', 'family_guid': 'F000005_5', 'project_guid': 'R0001_1kg', 'affected': 'N', 'sample_type': 'WES', -} -ALL_AFFECTED_SAMPLE_DATA['SNV_INDEL'].append(FAMILY_5_SAMPLE) -FAMILY_11_SAMPLE_WGS = { - 'sample_id': 'NA20885', 'individual_guid': 'I000015_na20885', 'family_guid': 'F000011_11', 'project_guid': 'R0003_test', 'affected': 'A', 'sample_type': 'WGS', -} -MULTI_PROJECT_SAMPLE_DATA = deepcopy(FAMILY_2_VARIANT_SAMPLE_DATA) -MULTI_PROJECT_SAMPLE_DATA['SNV_INDEL'].append({**FAMILY_11_SAMPLE_WGS, 'sample_type': 'WES'}) - -FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA = deepcopy(FAMILY_2_VARIANT_SAMPLE_DATA) -FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA['SNV_INDEL'].extend([ - {**s, 'sample_type': 'WGS'} for s in FAMILY_2_VARIANT_SAMPLE_DATA['SNV_INDEL']] -) - -FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA_MISSING_PARENTAL_WGS = deepcopy(FAMILY_2_VARIANT_SAMPLE_DATA) -FAMILY_2_BOTH_SAMPLE_TYPE_SAMPLE_DATA_MISSING_PARENTAL_WGS['SNV_INDEL'].extend([ - {**s, 'sample_type': 'WGS'} for s in FAMILY_2_VARIANT_SAMPLE_DATA['SNV_INDEL'] if s['sample_id'] == 'HG00731'] -) - -MULTI_PROJECT_SAMPLE_TYPES_SAMPLE_DATA = deepcopy(FAMILY_2_VARIANT_SAMPLE_DATA) -MULTI_PROJECT_SAMPLE_TYPES_SAMPLE_DATA['SNV_INDEL'].append(FAMILY_11_SAMPLE_WGS) -MULTI_PROJECT_SAMPLE_TYPES_SAMPLE_DATA['SNV_INDEL'].append({**FAMILY_11_SAMPLE_WGS, 'sample_type': 'WES'}) - -MULTI_PROJECT_MISSING_SAMPLE_DATA = deepcopy(FAMILY_2_MISSING_SAMPLE_DATA) -MULTI_PROJECT_MISSING_SAMPLE_DATA['SNV_INDEL'].append(FAMILY_11_SAMPLE_WGS) - -SV_WGS_SAMPLE_DATA_WITH_SEX = {'SV_WGS': [{'is_male': True, **FAMILY_11_SAMPLE_WGS}, { - 'sample_id': 'NA20884', 'individual_guid': 'I000025_na20884', 'family_guid': 'F000011_11', 'project_guid': 'R0003_test', 'affected': 'N', 'sample_type': 'WGS', 'is_male': True, -}, { - 'sample_id': 'NA20883', 'individual_guid': 'I000035_na20883', 'family_guid': 'F000011_11', 'project_guid': 'R0003_test', 'affected': 'N', 'sample_type': 'WGS', 'is_male': False, -}]} -SV_WGS_SAMPLE_DATA = deepcopy(SV_WGS_SAMPLE_DATA_WITH_SEX) -for s in SV_WGS_SAMPLE_DATA['SV_WGS']: - s.pop('is_male') - -SV_WES_SAMPLE_DATA = {'SV_WES': EXPECTED_SAMPLE_DATA['SV_WES'] + [FAMILY_3_SAMPLE]} - -VARIANT1 = { - 'variantId': '1-10439-AC-A', - 'chrom': '1', - 'pos': 10439, - 'ref': 'AC', - 'alt': 'A', - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '1', - 'liftedOverPos': 10439, - 'xpos': 1000010439, - 'rsid': 'rs112766696', - 'familyGuids': ['F000002_2'], - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'dp': 10, 'gq': 99, 'ab': 0.5, 'filters': [], - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'dp': 24, 'gq': 0, 'ab': 0.0, 'filters': [], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'dp': 60, 'gq': 20, 'ab': 0.0, 'filters': [], - }, - }, - 'clinvar': { - 'alleleId': 19473, - 'conflictingPathogenicities': None, - 'goldStars': None, - 'pathogenicity': 'Likely_pathogenic', - 'assertions': None, - 'submitters': None, - 'conditions': None, - 'version': '2024-02-21', - }, - 'hgmd': None, - 'screenRegionType': None, - 'populations': { - 'seqr': {'ac': 9, 'hom': 2}, - 'topmed': {'af': 0.0784199982881546, 'ac': 20757, 'an': 264690, 'hom': 0, 'het': 20757}, - 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, - 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, - 'gnomad_genomes': {'af': 0.034449315071105957, 'ac': 927, 'an': 26912, 'hom': 48, 'hemi': 0, 'filter_af': 0.040276646614074707}, - }, - 'predictions': { - 'cadd': 4.668000221252441, - 'eigen': None, - 'fathmm': None, - 'gnomad_noncoding': None, - 'mpc': None, - 'mut_pred': None, - 'primate_ai': None, - 'splice_ai': None, - 'splice_ai_consequence': None, - 'vest': None, - 'mut_taster': None, - 'polyphen': None, - 'revel': None, - 'sift': None, - }, - 'transcripts': {}, - 'sortedMotifFeatureConsequences': None, - 'sortedRegulatoryFeatureConsequences': None, - 'mainTranscriptId': None, - 'selectedMainTranscriptId': None, - '_sort': [1000010439], - 'CAID': 'CA16717152', -} - -VARIANT1_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY = deepcopy(VARIANT1) -genotypes = VARIANT1_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] -VARIANT1_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] = { - 'I000004_hg00731': [ - genotypes['I000004_hg00731'], - {**genotypes['I000004_hg00731'], 'numAlt': 2, 'sampleType': 'WGS'} - ], - 'I000005_hg00732': [genotypes['I000005_hg00732']], - 'I000006_hg00733': [genotypes['I000006_hg00733']], -} - -VARIANT1_BOTH_SAMPLE_TYPES = deepcopy(VARIANT1_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY) -genotypes = VARIANT1_BOTH_SAMPLE_TYPES['genotypes'] -VARIANT1_BOTH_SAMPLE_TYPES['genotypes']['I000005_hg00732'] = [ - *genotypes['I000005_hg00732'], - {**genotypes['I000005_hg00732'][0], 'gq': 99, 'numAlt': 1, 'sampleType': 'WGS'} -] -VARIANT1_BOTH_SAMPLE_TYPES['genotypes']['I000006_hg00733'] = [ - *genotypes['I000006_hg00733'], - {**genotypes['I000006_hg00733'][0], 'sampleType': 'WGS'} -] - -VARIANT2 = { - 'variantId': '1-38724419-T-G', - 'chrom': '1', - 'pos': 38724419, - 'ref': 'T', - 'alt': 'G', - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '1', - 'liftedOverPos': 11854476, - 'xpos': 1038724419, - 'rsid': 'rs1801131', - 'familyGuids': ['F000002_2'], - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 2, 'dp': 36, 'gq': 99, 'ab': 1.0, 'filters': [], - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'dp': 32, 'gq': 99, 'ab': 0.625, 'filters': [], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'dp': 33, 'gq': 40, 'ab': 0.0, 'filters': [], - }, - }, - 'clinvar': { - 'alleleId': 18560, - 'conflictingPathogenicities': [ - {'count': 1, 'pathogenicity': 'Likely_pathogenic'}, - {'count': 1, 'pathogenicity': 'Uncertain_significance'}, - {'count': 1, 'pathogenicity': 'Likely_benign'}, - {'count': 5, 'pathogenicity': 'Benign'}, - ], - 'goldStars': 1, - 'pathogenicity': 'Conflicting_classifications_of_pathogenicity', - 'assertions': ['other'], - 'version': '2024-02-21', - 'submitters': [ - 'Broad Center for Mendelian Genomics, Broad Institute of MIT and Harvard', - 'Illumina Laboratory Services, Illumina', - 'Blueprint Genetics', - 'GenomeConnect, ClinGen' - ], - 'conditions': [ - 'ABCA4-Related Disorders', - 'Severe early-childhood-onset retinal dystrophy', - 'not specified', - 'not provided' - ], - }, - 'hgmd': {'accession': 'CM981315', 'class': 'DFP'}, - 'screenRegionType': None, - 'populations': { - 'seqr': {'ac': 28, 'hom': 4}, - 'topmed': {'af': 0.24615199863910675, 'ac': 65154, 'an': 264690, 'hom': 8775, 'het': 47604}, - 'exac': {'af': 0.29499998688697815, 'ac': 35805, 'an': 121372, 'hom': 5872, 'hemi': 0, 'het': 24061, 'filter_af': 0.4153035283088684}, - 'gnomad_exomes': {'af': 0.28899794816970825, 'ac': 72672, 'an': 251462, 'hom': 11567, 'hemi': 0, 'filter_af': 0.4116474986076355}, - 'gnomad_genomes': {'af': 0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0}, - }, - 'predictions': { - 'cadd': 20.899999618530273, - 'eigen': 2.000999927520752, - 'fathmm': 0, - 'gnomad_noncoding': 5.868505001068115, - 'mpc': 0.28205373883247375, - 'mut_pred': None, - 'primate_ai': 0.4655807614326477, - 'splice_ai': 0.0, - 'splice_ai_consequence': 'No consequence', - 'vest': 0.210999995470047, - 'mut_taster': 'P', - 'polyphen': 0.1, - 'revel': 0.19699999690055847, - 'sift': 0.05, - }, - 'transcripts': { - 'ENSG00000177000': [ - {'aminoAcids': 'L/F', 'canonical': 1, 'codons': 'ttA/ttC', 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000383791.8:c.156A>C', 'hgvsp': 'ENSP00000373301.3:p.Leu52Phe', 'transcriptId': 'ENST00000383791', 'maneSelect': 'NM_004844.5', 'manePlusClinical': None, 'exon': {'index': 2, 'total': 9}, 'intron': None, 'alphamissense': {'pathogenicity': 0.9977999925613403}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_004844.5', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 0}, - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000408919.7:c.-384A>C', 'hgvsp': None, 'transcriptId': 'ENST00000408919', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 9}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'refseqTranscriptId': 'NM_001018009.4', 'biotype': 'protein_coding', 'majorConsequence': '5_prime_UTR_variant', 'consequenceTerms': ['5_prime_UTR_variant'], 'transcriptRank': 1, 'utrannotator': { - 'existingInframeOorfs': 0, 'existingOutofframeOorfs': 1, 'existingUorfs': 10, 'fiveutrConsequence': '5_prime_UTR_stop_codon_loss_variant', - 'fiveutrAnnotation': {'type': None, 'KozakContext': 'GCGATGC', 'KozakStrength': 'Moderate', 'DistanceToCDS': None, 'CapDistanceToStart': None, 'DistanceToStop': None, 'Evidence': False, 'AltStop': 'True', 'AltStopDistanceToCDS': 310, 'FrameWithCDS': 'outOfFrame', 'StartDistanceToCDS': None, 'newSTOPDistanceToCDS': None, 'alt_type': None, 'alt_type_length': None,'ref_StartDistanceToCDS': None, 'ref_type': None, 'ref_type_length': None}, - }}, - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000426925.5:c.-677A>C', 'hgvsp': None, 'transcriptId': 'ENST00000426925', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 11}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'refseqTranscriptId': None, 'biotype': 'protein_coding', 'majorConsequence': '5_prime_UTR_variant', 'consequenceTerms': ['5_prime_UTR_variant'], 'transcriptRank': 2, 'utrannotator': { - 'existingInframeOorfs': 0, 'existingOutofframeOorfs': 1, 'existingUorfs': 8,'fiveutrConsequence': '5_prime_UTR_stop_codon_loss_variant', - 'fiveutrAnnotation': {'type': None, 'KozakContext': 'TCAATGC', 'KozakStrength': 'Weak', 'DistanceToCDS': None, 'CapDistanceToStart': None, 'DistanceToStop': None, 'Evidence': False, 'AltStop': 'True', 'AltStopDistanceToCDS': 588, 'FrameWithCDS': 'inFrame', 'StartDistanceToCDS': None, 'newSTOPDistanceToCDS': None, 'alt_type': None, 'alt_type_length': None, 'ref_StartDistanceToCDS': None, 'ref_type': None, 'ref_type_length': None}, - }}, - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000412806.1:c.138+1722A>C', 'hgvsp': None, 'transcriptId': 'ENST00000412806', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 3},'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'nonsense_mediated_decay', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 3}, - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000459627.1:n.298A>C', 'hgvsp': None, 'transcriptId': 'ENST00000459627', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 3}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding_CDS_not_defined', 'majorConsequence': 'non_coding_transcript_exon_variant', 'consequenceTerms': ['non_coding_transcript_exon_variant'], 'transcriptRank': 4}, - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000465894.6:n.33A>C', 'hgvsp': None, 'transcriptId': 'ENST00000465894', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 5}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding_CDS_not_defined', 'majorConsequence': 'non_coding_transcript_exon_variant', 'consequenceTerms': ['non_coding_transcript_exon_variant'], 'transcriptRank': 5}, - ], - 'ENSG00000277258': [ - {'aminoAcids': 'L/F', 'canonical': None, 'codons': 'ttA/ttC', 'geneId': 'ENSG00000277258', 'hgvsc': 'ENST00000450625.1:c.156A>C', 'hgvsp': 'ENSP00000389484.1:p.Leu52Phe', 'transcriptId': 'ENST00000450625', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 2, 'total': 5}, 'intron': None, 'alphamissense': {'pathogenicity': 0.9977999925613403}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'nonsense_mediated_decay', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant', 'NMD_transcript_variant'], 'transcriptRank': 0}, - ] - }, - 'mainTranscriptId': 'ENST00000383791', - 'sortedMotifFeatureConsequences': None, - 'sortedRegulatoryFeatureConsequences': None, - 'selectedMainTranscriptId': None, - '_sort': [1038724419], - 'CAID': None, -} - -VARIANT2_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY = deepcopy(VARIANT2) -genotypes = VARIANT2_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] -VARIANT2_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] = { - 'I000004_hg00731': [ - genotypes['I000004_hg00731'], - {**genotypes['I000004_hg00731'], 'sampleType': 'WGS'} - ], - 'I000005_hg00732': [genotypes['I000005_hg00732']], - 'I000006_hg00733': [genotypes['I000006_hg00733']], -} - -VARIANT2_BOTH_SAMPLE_TYPES = deepcopy(VARIANT2_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY) -genotypes = VARIANT2_BOTH_SAMPLE_TYPES['genotypes'] -VARIANT2_BOTH_SAMPLE_TYPES['genotypes']['I000005_hg00732'] = [ - *genotypes['I000005_hg00732'], - {**genotypes['I000005_hg00732'][0], 'numAlt': 0, 'sampleType': 'WGS'} -] -VARIANT2_BOTH_SAMPLE_TYPES['genotypes']['I000006_hg00733'] = [ - *genotypes['I000006_hg00733'], - {**genotypes['I000006_hg00733'][0], 'sampleType': 'WGS'} -] - -VARIANT3 = { - 'variantId': '1-91502721-G-A', - 'chrom': '1', - 'pos': 91502721, - 'ref': 'G', - 'alt': 'A', - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '1', - 'liftedOverPos': 91968278, - 'xpos': 1091502721, - 'rsid': 'rs13447464', - 'familyGuids': ['F000002_2'], - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'dp': 40, 'gq': 99, 'ab': 1.0, 'filters': [], - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'dp': 37, 'gq': 99, 'ab': 0.4594594594594595, 'filters': [], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'dp': 27, 'gq': 99, 'ab': 0.4074074074074074, 'filters': [], - }, - }, - 'clinvar': None, - 'hgmd': None, - 'screenRegionType': None, - 'populations': { - 'seqr': {'ac': 4, 'hom': 1}, - 'topmed': {'af': 0.36268100142478943, 'ac': 95998, 'an': 264690, 'hom': 19369, 'het': 57260}, - 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, - 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, - 'gnomad_genomes': {'af': 0.38041073083877563, 'ac': 57757, 'an': 151828, 'hom': 12204, 'hemi': 0, 'filter_af': 0.4797786474227905}, - }, - 'predictions': { - 'cadd': 2.753999948501587, - 'eigen': 1.378000020980835, - 'fathmm': None, - 'gnomad_noncoding': 0.7389647960662842, - 'mpc': None, - 'mut_pred': None, - 'primate_ai': None, - 'splice_ai': 0.009999999776482582, - 'splice_ai_consequence': 'Donor gain', - 'vest': None, - 'mut_taster': None, - 'polyphen': None, - 'revel': None, - 'sift': None, - }, - 'transcripts': { - 'ENSG00000097046': [ - {'aminoAcids': None, 'canonical': 1, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000234626.11:c.-63-251G>A', 'hgvsp': None, 'transcriptId': 'ENST00000234626', 'maneSelect': 'NM_003503.4', 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 11}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_003503.4', 'biotype': 'protein_coding', 'majorConsequence': 'intron_variant', 'consequenceTerms': ['intron_variant'], 'transcriptRank': 0}, - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000428239.5:c.-64+100G>A', 'hgvsp': None, 'transcriptId': 'ENST00000428239', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 11}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_001134420.2', 'biotype': 'protein_coding', 'majorConsequence': 'intron_variant', 'consequenceTerms': ['intron_variant'], 'transcriptRank': 1}, - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000497611.1:n.244G>A', 'hgvsp': None, 'transcriptId': 'ENST00000497611', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 1, 'total': 4}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding_CDS_not_defined', 'majorConsequence': 'non_coding_transcript_exon_variant', 'consequenceTerms': ['non_coding_transcript_exon_variant'], 'transcriptRank': 2}, - ], - 'ENSG00000177000': [ - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000177000', 'hgvsc': 'ENST00000426137.1:c.-64+100G>A', 'hgvsp': None, 'transcriptId': 'ENST00000426137', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 5}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding', 'majorConsequence': 'intron_variant', 'consequenceTerms': ['intron_variant'], 'transcriptRank': 0}, - ], - }, - 'mainTranscriptId': 'ENST00000234626', - 'sortedMotifFeatureConsequences': None, - 'sortedRegulatoryFeatureConsequences': [{'biotype': 'promoter', 'consequenceTerms': ['regulatory_region_variant'], 'regulatoryFeatureId': 'ENSR00000009706'}], - 'selectedMainTranscriptId': None, - '_sort': [1091502721], - 'CAID': 'CA10960369', -} - -VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY = deepcopy(VARIANT3) -genotypes = VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] -VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] = { - 'I000004_hg00731': [ - genotypes['I000004_hg00731'], - {**genotypes['I000004_hg00731'], 'sampleType': 'WGS'} - ], - 'I000005_hg00732': [genotypes['I000005_hg00732']], - 'I000006_hg00733': [genotypes['I000006_hg00733']], -} - -VARIANT3_BOTH_SAMPLE_TYPES = deepcopy(VARIANT3_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY) -genotypes = VARIANT3_BOTH_SAMPLE_TYPES['genotypes'] -VARIANT3_BOTH_SAMPLE_TYPES['genotypes']['I000005_hg00732'] = [ - *genotypes['I000005_hg00732'], - {**genotypes['I000005_hg00732'][0], 'sampleType': 'WGS'} -] -VARIANT3_BOTH_SAMPLE_TYPES['genotypes']['I000006_hg00733'] = [ - *genotypes['I000006_hg00733'], - {**genotypes['I000006_hg00733'][0], 'sampleType': 'WGS'} -] - -VARIANT4 = { - 'variantId': '1-91511686-T-G', - 'chrom': '1', - 'pos': 91511686, - 'ref': 'T', - 'alt': 'G', - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '1', - 'liftedOverPos': 91977243, - 'xpos': 1091511686, - 'rsid': None, - 'familyGuids': ['F000002_2'], - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'dp': 29, 'gq': 58, 'ab': 0.1724137931034483, 'filters': ['VQSRTrancheSNP99.95to100.00'], - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'dp': 24, 'gq': 0, 'ab': 0.0, 'filters': ['VQSRTrancheSNP99.95to100.00'], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'dp': 45, 'gq': 0, 'ab': 0.0, 'filters': ['VQSRTrancheSNP99.95to100.00'], - }, - }, - 'clinvar': None, - 'hgmd': None, - 'screenRegionType': 'CTCF-only', - 'populations': { - 'seqr': {'ac': 2, 'hom': 0}, - 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, - 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, - 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, - 'gnomad_genomes': {'af': 0.00026519427774474025, 'ac': 39, 'an': 147062, 'hom': 0, 'hemi': 0, 'filter_af': 0.0015030059730634093}, - }, - 'predictions': { - 'cadd': 29.899999618530273, - 'eigen': 9.491000175476074, - 'fathmm': 0, - 'gnomad_noncoding': 0.2300506979227066, - 'mpc': 0.8326827883720398, - 'mut_pred': 0.6869999766349792, - 'primate_ai': 0.6995947360992432, - 'splice_ai': 0.0, - 'splice_ai_consequence': 'No consequence', - 'vest': 0.8579999804496765, - 'mut_taster': 'N', - 'polyphen': 0, - 'revel': 0.5260000228881836, - 'sift': 0, - }, - 'transcripts': { - 'ENSG00000097046': [ - {'aminoAcids': None, 'canonical': None, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000466716.5:c.-264+1G>A', 'hgvsp': None, 'transcriptId': 'ENST00000466716', 'maneSelect': None, 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 3}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': ['5UTR_SPLICE']}, 'spliceregion': {'extended_intronic_splice_region_variant': True}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': None, 'biotype': 'protein_coding', 'majorConsequence': 'splice_donor_variant', 'consequenceTerms': ['splice_donor_variant'], 'transcriptRank': 0}, - {'aminoAcids': None, 'canonical': 1, 'codons': None, 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000350997.12:c.375+139G>A', 'hgvsp': None, 'transcriptId': 'ENST00000350997', 'maneSelect': 'NM_013402.7', 'manePlusClinical': None, 'exon': None, 'intron': {'index': 1, 'total': 11}, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_013402.7', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 1}, - {'aminoAcids': 'T/I', 'canonical': None, 'codons': 'aCc/aTc', 'geneId': 'ENSG00000097046', 'hgvsc': 'ENST00000257261.10:c.131C>T', 'hgvsp': 'ENSP00000257261.6:p.Thr44Ile', 'transcriptId': 'ENST00000257261', 'maneSelect': None, 'manePlusClinical': None, 'exon': {'index': 1, 'total': 12}, 'intron': None, 'alphamissense': {'pathogenicity': None}, 'loftee': {'isLofNagnag': None, 'lofFilters': None}, 'spliceregion': {'extended_intronic_splice_region_variant': False}, 'utrannotator': {'existingInframeOorfs': None, 'existingOutofframeOorfs': None, 'existingUorfs': None, 'fiveutrAnnotation': None, 'fiveutrConsequence': None}, 'refseqTranscriptId': 'NM_001281501.1', 'biotype': 'protein_coding', 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant'], 'transcriptRank': 2}, - ], - }, - 'mainTranscriptId': 'ENST00000466716', - 'sortedMotifFeatureConsequences': [ - {'consequenceTerms': ['TF_binding_site_variant'], 'motifFeatureId': 'ENSM00093424674'}, - {'consequenceTerms': ['TF_binding_site_variant'], 'motifFeatureId': 'ENSM00036268032'}, - ], - 'sortedRegulatoryFeatureConsequences': [ - {'biotype': 'promoter', 'consequenceTerms': ['regulatory_region_variant'], 'regulatoryFeatureId': 'ENSR00000040341'}, - ], - 'selectedMainTranscriptId': None, - '_sort': [1091511686], - 'CAID': 'CA341062623', -} - -VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY = deepcopy(VARIANT4) -genotypes = VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] -VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY['genotypes'] = { - 'I000004_hg00731': [ - genotypes['I000004_hg00731'], - {**genotypes['I000004_hg00731'], 'sampleType': 'WGS'} - ], - 'I000005_hg00732': [genotypes['I000005_hg00732']], - 'I000006_hg00733': [genotypes['I000006_hg00733']], -} - -VARIANT4_BOTH_SAMPLE_TYPES = deepcopy(VARIANT4_BOTH_SAMPLE_TYPES_PROBAND_WGS_ONLY) -genotypes = VARIANT4_BOTH_SAMPLE_TYPES['genotypes'] -VARIANT4_BOTH_SAMPLE_TYPES['genotypes']['I000005_hg00732'] = [ - *genotypes['I000005_hg00732'], - {**genotypes['I000005_hg00732'][0], 'sampleType': 'WGS'} -] -VARIANT4_BOTH_SAMPLE_TYPES['genotypes']['I000006_hg00733'] = [ - *genotypes['I000006_hg00733'], - {**genotypes['I000006_hg00733'][0], 'numAlt': 2, 'sampleType': 'WGS'} -] - -VARIANT_LOOKUP_VARIANT = { - **VARIANT1, - 'liftedFamilyGuids': ['F000014_14'], - 'familyGenotypes': { - VARIANT1['familyGuids'][0]: sorted([ - {k: v for k, v in g.items() if k != 'individualGuid'} for g in VARIANT1['genotypes'].values() - ], key=lambda x: x['sampleId'], reverse=True), - 'F000011_11': [{ - 'sampleId': 'NA20885', 'sampleType': 'WES', 'familyGuid': 'F000011_11', - 'numAlt': 2, 'dp': 6, 'gq': 16, 'ab': 1.0, 'filters': [], - }], - 'F000014_14': [{ - 'sampleId': 'NA21234', 'sampleType': 'WGS', 'familyGuid': 'F000014_14', - 'numAlt': 1, 'dp': 27, 'gq': 87, 'ab': 0.531000018119812, 'filters': None, - }], - } -} -for k in {'familyGuids', 'genotypes'}: - VARIANT_LOOKUP_VARIANT.pop(k) - -FAMILY_3_VARIANT = deepcopy(VARIANT3) -FAMILY_3_VARIANT['familyGuids'] = ['F000003_3'] -FAMILY_3_VARIANT['genotypes'] = { - 'I000007_na20870': { - 'sampleId': 'NA20870', 'sampleType': 'WES', 'individualGuid': 'I000007_na20870', 'familyGuid': 'F000003_3', - 'numAlt': 1, 'dp': 28, 'gq': 99, 'ab': 0.6785714285714286, 'filters': [], - }, -} -MULTI_FAMILY_VARIANT = deepcopy(VARIANT3) -MULTI_FAMILY_VARIANT['familyGuids'] += FAMILY_3_VARIANT['familyGuids'] -MULTI_FAMILY_VARIANT['genotypes'].update(FAMILY_3_VARIANT['genotypes']) - -HAIL_BACKEND_VARIANTS = [VARIANT2, MULTI_FAMILY_VARIANT] -HAIL_BACKEND_SINGLE_FAMILY_VARIANTS = [VARIANT2, VARIANT3] - -PROJECT_2_VARIANT = { - 'variantId': '1-10146-ACC-A', - 'chrom': '1', - 'pos': 10146, - 'ref': 'ACC', - 'alt': 'A', - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '1', - 'liftedOverPos': 10146, - 'xpos': 1000010146, - 'rsid': 'rs375931351', - 'familyGuids': ['F000011_11'], - 'genotypes': { - 'I000015_na20885': { - 'sampleId': 'NA20885', 'sampleType': 'WES', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'dp': 8, 'gq': 14, 'ab': 0.875, 'filters': [], - } - }, - 'clinvar': None, - 'hgmd': None, - 'screenRegionType': None, - 'populations': { - 'seqr': {'ac': 0, 'hom': 0}, - 'topmed': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'het': 0}, - 'exac': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'het': 0, 'filter_af': 0.0}, - 'gnomad_exomes': {'af': 0.0, 'ac': 0, 'an': 0, 'hom': 0, 'hemi': 0, 'filter_af': 0.0}, - 'gnomad_genomes': {'af': 0.00012430080096237361, 'ac': 2, 'an': 16090, 'hom': 0, 'hemi': 0, 'filter_af': 0.002336448524147272}, - }, - 'predictions': { - 'cadd': 4.6529998779296875, - 'eigen': None, - 'fathmm': None, - 'gnomad_noncoding': None, - 'mpc': None, - 'mut_pred': None, - 'primate_ai': None, - 'splice_ai': None, - 'splice_ai_consequence': None, - 'vest': None, - 'mut_taster': None, - 'polyphen': None, - 'revel': None, - 'sift': None, - }, - 'transcripts': {}, - 'mainTranscriptId': None, - 'selectedMainTranscriptId': None, - 'sortedMotifFeatureConsequences': None, - 'sortedRegulatoryFeatureConsequences': None, - '_sort': [1000010146], - 'CAID': 'CA520798130', -} - -GRCH37_VARIANT = { - 'variantId': '7-143270172-A-G', - 'xpos': 7143270172, - 'chrom': '7', - 'pos': 143270172, - 'ref': 'A', - 'alt': 'G', - 'genomeVersion': '37', - 'rsid': 'rs72611576', - 'familyGuids': ['F000002_2'], - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WGS', 'individualGuid': 'I000004_hg00731', - 'familyGuid': 'F000002_2', 'numAlt': 2, 'dp': 16, 'gq': 48, 'ab': 1, 'filters': ['VQSRTrancheSNP99.90to99.95'], - }, 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000006_hg00733', - 'familyGuid': 'F000002_2', 'numAlt': 1, 'dp': 49, 'gq': 99, 'ab': 0.6530612111091614, - 'filters': ['VQSRTrancheSNP99.90to99.95'], - }, - }, - 'populations': { - 'seqr': {'ac': 4711, 'hom': 1508}, - 'topmed': {'af': 0.5213189721107483, 'ac': 65461, 'an': 125568, 'hom': 16156, 'het': 33149}, - 'exac': {'af': 0.6299999952316284, 'ac': 66593, 'an': 104352, 'hom': 22162, 'hemi': 0, 'het': 22269, 'filter_af': 0.8198773860931396}, - 'gnomad_exomes': {'af': 0.6354219317436218, 'ac': 137532, 'an': 216442, 'hom': 45869, 'hemi': 0, 'filter_af': 0.8226116299629211}, - 'gnomad_genomes': {'af': 0.6136477589607239, 'ac': 14649, 'an': 23872, 'hom': 4584, 'hemi': 0, 'filter_af': 0.828438937664032}, - }, - 'predictions': { - 'cadd': 13.020000457763672, 'eigen': 3.9509999752044678, 'primate_ai': 0.4906357526779175, - 'splice_ai': 0, 'splice_ai_consequence': 'No consequence', - 'mpc': None, 'mut_taster': None, 'polyphen': None, 'revel': None, 'sift': None, - }, - 'clinvar': None, - 'hgmd': None, - 'transcripts': { - 'ENSG00000271079': [ - {'aminoAcids': 'E/G', 'canonical': 1, 'codons': 'gAa/gGa', 'geneId': 'ENSG00000271079', - 'hgvsc': 'ENST00000420911.2:c.1262A>G', 'hgvsp': 'ENSP00000474204.1:p.Glu421Gly', - 'transcriptId': 'ENST00000420911', 'isLofNagnag': None, 'transcriptRank': 0, 'consequenceTerms': ['missense_variant'], - 'biotype': 'protein_coding', 'lofFilters': None, 'majorConsequence': 'missense_variant'}, - ], - 'ENSG00000176227': [ - {'aminoAcids': None, 'canonical': 1, 'codons': None, 'geneId': 'ENSG00000176227', - 'hgvsc': 'ENST00000447022.1:n.1354A>G', 'hgvsp': None, - 'transcriptId': 'ENST00000447022', 'isLofNagnag': None, 'transcriptRank': 0, - 'biotype': 'processed_pseudogene', 'lofFilters': None, 'majorConsequence': 'non_coding_transcript_exon_variant', - 'consequenceTerms': ['non_coding_transcript_exon_variant', 'non_coding_transcript_variant']}, - ], - }, - 'mainTranscriptId': 'ENST00000420911', - 'selectedMainTranscriptId': None, - '_sort': [7143270172], - 'CAID': 'CA4540310', -} - -SV_VARIANT1 = { - 'variantId': 'phase2_DEL_chr1_625', - 'chrom': '1', - 'endChrom': None, - 'pos': 9310023, - 'end': 9310127, - 'rg37LocusEnd': {'contig': '1', 'position': 9370186}, - 'xpos': 1009310023, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '1', - 'liftedOverPos': 9370082, - 'algorithms': 'manta', - 'bothsidesSupport': False, - 'familyGuids': ['F000011_11'], - 'genotypes': { - 'I000015_na20885': { - 'sampleId': 'NA20885', 'sampleType': 'WGS', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'cn': 2, 'gq': 0, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, 'I000025_na20884': { - 'sampleId': 'NA20884', 'sampleType': 'WGS', 'individualGuid': 'I000025_na20884', 'familyGuid': 'F000011_11', - 'numAlt': 0, 'cn': 2, 'gq': 6, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, 'I000035_na20883': { - 'sampleId': 'NA20883', 'sampleType': 'WGS', 'individualGuid': 'I000035_na20883', 'familyGuid': 'F000011_11', - 'numAlt': 0, 'cn': 2, 'gq': 99, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, - }, - 'populations': { - 'sv_callset': {'af': 0.02421800047159195, 'ac': 141, 'an': 5822, 'hom': 0, 'het': 141}, - 'gnomad_svs': {'af': 0.0, 'id': '', 'hom': 0, 'het': 0}, - }, - 'predictions': {'strvctvre': None}, - 'cpxIntervals': None, - 'svSourceDetail': None, - 'svType': 'DEL', - 'svTypeDetail': None, - 'transcripts': { - 'ENSG00000171621': [{'geneId': 'ENSG00000171621', 'majorConsequence': 'INTRONIC'}], - }, - '_sort': [1009310023], -} -SV_VARIANT2 = { - 'variantId': 'cohort_2911.chr1.final_cleanup_INS_chr1_160', - 'chrom': '1', - 'endChrom': None, - 'pos': 9380254, - 'end': 9380286, - 'rg37LocusEnd': {'contig': '4', 'position': 9382012}, - 'xpos': 1009380254, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '1', - 'liftedOverPos': 9440313, - 'algorithms': 'manta', - 'bothsidesSupport': True, - 'familyGuids': ['F000011_11'], - 'genotypes': { - 'I000015_na20885': { - 'sampleId': 'NA20885', 'sampleType': 'WGS', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'cn': None, 'gq': 0, 'newCall': True, 'prevCall': False, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, 'I000025_na20884': { - 'sampleId': 'NA20884', 'sampleType': 'WGS', 'individualGuid': 'I000025_na20884', 'familyGuid': 'F000011_11', - 'numAlt': 0, 'cn': None, 'gq': 99, 'newCall': True, 'prevCall': False, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, 'I000035_na20883': { - 'sampleId': 'NA20883', 'sampleType': 'WGS', 'individualGuid': 'I000035_na20883', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'cn': None, 'gq': 0, 'newCall': True, 'prevCall': False, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, - }, - 'populations': { - 'sv_callset': {'af': 0.199072003364563, 'ac': 1159, 'an': 5822, 'hom': 0, 'het': 1159}, - 'gnomad_svs': {'af': 0.005423000082373619, 'id': 'gnomAD-SV_v3_INS_1_299', 'hom': 10359, 'het': 35634}, - }, - 'predictions': {'strvctvre': None}, - 'cpxIntervals': None, - 'svSourceDetail': {'chrom': '4'}, - 'svType': 'INS', - 'svTypeDetail': None, - 'transcripts': { - 'ENSG00000171621': [{'geneId': 'ENSG00000171621', 'majorConsequence': 'NEAREST_TSS'}], - }, - '_sort': [1009380254], -} -SV_VARIANT3 = { - 'variantId': 'phase2_CPX_chr13_35', - 'chrom': '13', - 'endChrom': '17', - 'pos': 63036002, - 'end': 63036041, - 'rg37LocusEnd': {'contig': '17', 'position': 61113402}, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '13', - 'liftedOverPos': 63610135, - 'xpos': 13063036002, - 'algorithms': 'manta', - 'bothsidesSupport': False, - 'familyGuids': ['F000011_11'], - 'genotypes': { - 'I000015_na20885': { - 'sampleId': 'NA20885', 'sampleType': 'WGS', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'cn': None, 'gq': 62, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, 'I000025_na20884': { - 'sampleId': 'NA20884', 'sampleType': 'WGS', 'individualGuid': 'I000025_na20884', 'familyGuid': 'F000011_11', - 'numAlt': 2, 'cn': None, 'gq': 42, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, 'I000035_na20883': { - 'sampleId': 'NA20883', 'sampleType': 'WGS', 'individualGuid': 'I000035_na20883', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'cn': None, 'gq': 79, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': ['HIGH_SR_BACKGROUND'], - }, - }, 'populations': { - 'sv_callset': {'af': 0.5778080224990845, 'ac': 3364, 'an': 5822, 'hom': 459, 'het': 2446}, - 'gnomad_svs': {'af': 0.0, 'id': '', 'hom': 0, 'het': 0}, - }, - 'predictions': {'strvctvre': None}, - 'cpxIntervals': [{'chrom': 'chr17', 'start': 22150735, 'end': 22151179, 'type': 'DUP'}], - 'svSourceDetail': None, - 'svType': 'CPX', - 'svTypeDetail': 'dDUP', - 'transcripts': { - 'ENSG00000083544': [{'geneId': 'ENSG00000083544', 'majorConsequence': 'NEAREST_TSS'}], - 'null': [{'geneId': None, 'majorConsequence': 'NEAREST_TSS'}], - }, - '_sort': [13063036002], -} -SV_VARIANT4 = { - 'variantId': 'phase2_DEL_chr14_4640', - 'chrom': '14', - 'endChrom': None, - 'pos': 106694244, - 'end': 106740587, - 'rg37LocusEnd': None, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '14', - 'liftedOverPos': 107150261, - 'xpos': 14106694244, - 'algorithms': 'depth,manta', - 'bothsidesSupport': False, - 'familyGuids': ['F000011_11'], - 'genotypes': { - 'I000015_na20885': { - 'sampleId': 'NA20885', 'sampleType': 'WGS', 'individualGuid': 'I000015_na20885', 'familyGuid': 'F000011_11', - 'numAlt': 2, 'cn': 0, 'gq': 99, 'newCall': False, 'prevCall': True, 'prevNumAlt': None, 'filters': [], - }, 'I000025_na20884': { - 'sampleId': 'NA20884', 'sampleType': 'WGS', 'individualGuid': 'I000025_na20884', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'cn': 1, 'gq': 99, 'newCall': False, 'prevCall': False, 'prevNumAlt': 2, 'filters': [], - }, 'I000035_na20883': { - 'sampleId': 'NA20883', 'sampleType': 'WGS', 'individualGuid': 'I000035_na20883', 'familyGuid': 'F000011_11', - 'numAlt': 1, 'cn': 1, 'gq': 99, 'newCall': False, 'prevCall': False, 'prevNumAlt': 2, 'filters': [], - }, - }, - 'populations': { - 'sv_callset': {'af': 0.6100999712944031, 'ac': 3552, 'an': 5822, 'hom': 1053, 'het': 1446}, - 'gnomad_svs': {'af': 0.0, 'id': '', 'hom': 0, 'het': 0}, - }, - 'predictions': {'strvctvre': 0.16099999845027924}, - 'cpxIntervals': None, - 'svSourceDetail': None, - 'svType': 'DEL', - 'svTypeDetail': None, - 'transcripts': { - 'ENSG00000184986': [{'geneId': 'ENSG00000184986', 'majorConsequence': 'NEAREST_TSS'}], - }, - '_sort': [14106694244], -} -GCNV_VARIANT1 = { - 'variantId': 'suffix_95340_DUP', - 'chrom': '14', - 'pos': 22438910, - 'end': 22469796, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '14', - 'liftedOverPos': 22886546, - 'rg37LocusEnd': {'contig': '14', 'position': 23058228}, - 'xpos': 14022417556, - 'familyGuids': ['F000002_2'], - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'cn': 3, 'qs': 38, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'cn': None, 'qs': None, 'defragged': None, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': None, 'prevCall': None, 'prevOverlap': None, 'filters': [], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'cn': None, 'qs': None, 'defragged': None, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': None, 'prevCall': None, 'prevOverlap': None, 'filters': [], - } - }, - 'populations': {'sv_callset': {'af': 0.076492540538311, 'ac': 1763, 'an': 23048, 'hom': 0, 'het': 0}}, - 'predictions': {'strvctvre': 0.1809999942779541}, - 'numExon': 0, - 'svType': 'DUP', - 'transcripts': {}, - '_sort': [14022417556], - -} -GCNV_VARIANT2 = { - 'variantId': 'suffix_124520_DUP', - 'chrom': '16', - 'pos': 29809156, - 'end': 29815990, - 'xpos': 16029802672, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '16', - 'liftedOverPos': 29813993, - 'rg37LocusEnd': {'contig': '16', 'position': 29831761}, - 'familyGuids': ['F000002_2'], - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'cn': 3, 'qs': 29, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'cn': 3, 'qs': 46, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'cn': 3, 'qs': 37, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], - } - }, - 'populations': {'sv_callset': {'af': 0.012322110123932362, 'ac': 284, 'an': 23047, 'hom': 0, 'het': 0}}, - 'predictions': {'strvctvre': 0.5479999780654907}, - 'numExon': 8, - 'svType': 'DUP', - 'transcripts': { - 'ENSG00000103495': [{'geneId': 'ENSG00000103495', 'majorConsequence': 'COPY_GAIN'}], - 'ENSG00000167371': [{'geneId': 'ENSG00000167371', 'majorConsequence': 'COPY_GAIN'}], - 'ENSG00000280893': [{'geneId': 'ENSG00000280893', 'majorConsequence': 'COPY_GAIN'}], - }, - '_sort': [16029802672], -} -GCNV_VARIANT3 = { - 'variantId': 'suffix_140593_DUP', - 'chrom': '17', - 'pos': 38717327, - 'end': 38719636, - 'xpos': 17038717327, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '17', - 'liftedOverPos': 36873580, - 'rg37LocusEnd': {'contig': '17', 'position': 36876246}, - 'familyGuids': ['F000002_2'], - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 2, 'cn': 4, 'qs': 13, 'defragged': True, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': True, 'prevCall': False, 'prevOverlap': False, 'filters': [], - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'cn': 3, 'qs': 7, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': False, 'prevOverlap': True, 'filters': [], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', 'numAlt': 0, - 'cn': None, 'qs': None, 'defragged': None, 'start': None, 'end': None, 'numExon': None, 'geneIds': None, - 'newCall': None, 'prevCall': None, 'prevOverlap': None, 'filters': [], - }, - }, - 'populations': {'sv_callset': {'af': 0.0015185698866844177, 'ac': 35, 'an': 23048, 'hom': 0, 'het': 0}}, - 'predictions': {'strvctvre': 0.7860000133514404}, - 'numExon': 3, - 'svType': 'DUP', - 'transcripts': { - 'ENSG00000275023': [{'geneId': 'ENSG00000275023', 'majorConsequence': 'LOF'}], - }, - '_sort': [17038717327], -} -GCNV_VARIANT4 = { - 'variantId': 'suffix_140608_DUP', - 'chrom': '17', - 'pos': 38721781, - 'end': 38735703, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': '17', - 'liftedOverPos': 36878034, - 'rg37LocusEnd': {'contig': '17', 'position': 36892521}, - 'familyGuids': ['F000002_2'], - 'xpos': 17038721781, - 'genotypes': { - 'I000004_hg00731': { - 'sampleId': 'HG00731', 'sampleType': 'WES', 'individualGuid': 'I000004_hg00731', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'cn': 3, 'qs': 28, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], - }, - 'I000005_hg00732': { - 'sampleId': 'HG00732', 'sampleType': 'WES', 'individualGuid': 'I000005_hg00732', 'familyGuid': 'F000002_2', - 'numAlt': 0, 'cn': None, 'qs': None, 'defragged': None, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': None, 'prevCall': None, 'prevOverlap': None, 'filters': [], - }, - 'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WES', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'cn': 3, 'qs': 29, 'defragged': False, 'start': None, 'end': 38734440, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], - } - }, - 'populations': {'sv_callset': {'af': 0.004989586770534515, 'ac': 115, 'an': 23048, 'hom': 0, 'het': 0}}, - 'predictions': {'strvctvre': 0.7099999785423279}, - 'numExon': 7, - 'svType': 'DEL', - 'transcripts': { - 'ENSG00000275023': [{'geneId': 'ENSG00000275023', 'majorConsequence': 'LOF'}], - 'ENSG00000277258': [{'geneId': 'ENSG00000277258', 'majorConsequence': 'LOF'}], - 'ENSG00000277972': [{'geneId': 'ENSG00000277972', 'majorConsequence': 'COPY_GAIN'}], - }, - '_sort': [17038721781], -} - -GCNV_MULTI_FAMILY_VARIANT1 = deepcopy(GCNV_VARIANT1) -GCNV_MULTI_FAMILY_VARIANT1.update({ - 'pos': 22418039, - 'end': 22507821, - 'transcripts': { - 'ENSG00000129562': [{'geneId': 'ENSG00000129562', 'majorConsequence': 'COPY_GAIN'}], - }, -}) -GCNV_MULTI_FAMILY_VARIANT1['familyGuids'].append('F000003_3') -GCNV_MULTI_FAMILY_VARIANT1['genotypes'].update({'I000007_na20870': { - 'sampleId': 'NA20870', 'sampleType': 'WES', 'individualGuid': 'I000007_na20870', 'familyGuid': 'F000003_3', - 'numAlt': 1, 'cn': 3, 'qs': 164, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], -}}) -GCNV_MULTI_FAMILY_VARIANT1['genotypes']['I000004_hg00731'].update({'start': 22438910, 'end': 22469796, 'geneIds': []}) - -GCNV_MULTI_FAMILY_VARIANT2 = deepcopy(GCNV_VARIANT2) -GCNV_MULTI_FAMILY_VARIANT2['numExon'] = 26 -GCNV_MULTI_FAMILY_VARIANT2['familyGuids'].append('F000003_3') -for genotype in GCNV_MULTI_FAMILY_VARIANT2['genotypes'].values(): - genotype.update({'numExon': 8, 'geneIds': ['ENSG00000103495', 'ENSG00000167371', 'ENSG00000280893']}) -GCNV_MULTI_FAMILY_VARIANT2['genotypes'].update({'I000007_na20870': { - 'sampleId': 'NA20870', 'sampleType': 'WES', 'individualGuid': 'I000007_na20870', 'familyGuid': 'F000003_3', - 'numAlt': 1, 'cn': 3, 'qs': 40, 'defragged': False, 'start': None, 'end': None, 'numExon': None, - 'geneIds': None, 'newCall': False, 'prevCall': True, 'prevOverlap': False, 'filters': [], -}}) -GCNV_MULTI_FAMILY_VARIANT2['transcripts'].update({ - 'ENSG00000013364': [{'geneId': 'ENSG00000013364', 'majorConsequence': 'LOF'}], - 'ENSG00000079616': [{'geneId': 'ENSG00000079616', 'majorConsequence': 'LOF'}], - 'ENSG00000281348': [{'geneId': 'ENSG00000281348', 'majorConsequence': 'LOF'}], - 'ENSG00000280789': [{'geneId': 'ENSG00000280789', 'majorConsequence': 'LOF'}], -}) - -MITO_VARIANT1 = { - 'variantId': 'M-4429-G-A', - 'chrom': 'M', - 'pos': 4429, - 'ref': 'G', - 'alt': 'A', - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': 'MT', - 'liftedOverPos': 4429, - 'xpos': 25000004429, - 'rsid': 'rs1603219456', - 'familyGuids': ['F000002_2'], - 'genotypes': {'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'dp': 3955, 'hl': 0.083, 'mitoCn': 230, 'contamination': 0.0, 'filters': [], - }}, - 'populations': { - 'seqr': {'af': 0.0, 'ac': 0, 'an': 154}, - 'seqr_heteroplasmy': {'af': 0.006493506487458944, 'ac': 1, 'an': 154}, - 'gnomad_mito': {'af': 0.0, 'ac': 0, 'an': 56419}, - 'gnomad_mito_heteroplasmy': {'af': 0.0, 'ac': 0, 'an': 56419, 'max_hl': 0.0}, - 'helix': {'af': 0.0, 'ac': 0, 'an': 195983}, - 'helix_heteroplasmy': {'af': 1.530745066702366e-05, 'ac': 3, 'an': 195983, 'max_hl': 0.20634999871253967}, - }, - 'predictions': { - 'apogee': None, - 'haplogroup_defining': None, - 'hmtvar': 0.05000000074505806, - 'mitotip': 'likely_pathogenic', - 'mut_taster': None, - 'sift': None, - 'mlc': 3.38874, - }, - 'commonLowHeteroplasmy': False, - 'mitomapPathogenic': None, - 'clinvar': { - 'alleleId': 677748, - 'conflictingPathogenicities': None, - 'goldStars': 1, - 'pathogenicity': 'Uncertain_significance', - 'assertions': [], - 'version': '2024-02-21', - }, - 'transcripts': { - 'ENSG00000210112': [ - {'aminoAcids': None, 'canonical': 1, 'codons': None, 'geneId': 'ENSG00000210112', 'hgvsc': 'ENST00000387377.1:n.28G>A', 'hgvsp': None, 'transcriptId': 'ENST00000387377', 'isLofNagnag': None, 'transcriptRank': 0, 'biotype': 'Mt_tRNA', 'lofFilters': None, 'majorConsequence': 'non_coding_transcript_exon_variant', 'consequenceTerms': ['non_coding_transcript_exon_variant']}, - ], - }, - 'mainTranscriptId': 'ENST00000387377', - 'selectedMainTranscriptId': None, - '_sort': [25000004429], -} -MITO_VARIANT2 = { - 'variantId': 'M-11597-A-G', - 'chrom': 'M', - 'pos': 11597, - 'ref': 'A', - 'alt': 'G', - 'xpos': 25000011597, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': 'MT', - 'liftedOverPos': 11597, - 'rsid': None, - 'familyGuids': ['F000002_2'], - 'genotypes': {'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 1, 'dp': 3845, 'hl': 0.029, 'mitoCn': 247, 'contamination': 0.015, 'filters': [], - }}, - 'populations': { - 'seqr': {'af': 0.0, 'ac': 0, 'an': 154}, - 'seqr_heteroplasmy': {'af': 0.006493506487458944, 'ac': 1, 'an': 154}, - 'gnomad_mito': {'af': 0.0, 'ac': 0, 'an': 0}, - 'gnomad_mito_heteroplasmy': {'af': 0.0, 'ac': 0, 'an': 0, 'max_hl': 0.0}, - 'helix': {'af': 0.0, 'ac': 0, 'an': 0}, - 'helix_heteroplasmy': {'af': 0.0, 'ac': 0, 'an': 0, 'max_hl': 0.0}, - }, - 'predictions': { - 'apogee': 0.5799999833106995, - 'haplogroup_defining': None, - 'hmtvar': 0.75, - 'mitotip': None, - 'mut_taster': 'N', - 'sift': 0, - 'mlc': None, - }, - 'commonLowHeteroplasmy': False, - 'mitomapPathogenic': None, - 'clinvar': None, - 'transcripts': { - 'ENSG00000198886': [ - {'aminoAcids': 'T/A', 'canonical': 1, 'codons': 'Aca/Gca', 'geneId': 'ENSG00000198886', 'hgvsc': 'ENST00000361381.2:c.838A>G', 'hgvsp': 'ENSP00000354961.2:p.Thr280Ala', 'transcriptId': 'ENST00000361381', 'isLofNagnag': None, 'transcriptRank': 0, 'biotype': 'protein_coding', 'lofFilters': None, 'majorConsequence': 'missense_variant', 'consequenceTerms': ['missense_variant']}, - ], - }, - 'mainTranscriptId': 'ENST00000361381', - 'selectedMainTranscriptId': None, - '_sort': [25000011597], -} -MITO_VARIANT3 = { - 'variantId': 'M-14783-T-C', - 'chrom': 'M', - 'pos': 14783, - 'ref': 'T', - 'alt': 'C', - 'xpos': 25000014783, - 'genomeVersion': '38', - 'liftedOverGenomeVersion': '37', - 'liftedOverChrom': 'MT', - 'liftedOverPos': 14783, - 'rsid': 'rs193302982', - 'familyGuids': ['F000002_2'], - 'genotypes': {'I000006_hg00733': { - 'sampleId': 'HG00733', 'sampleType': 'WGS', 'individualGuid': 'I000006_hg00733', 'familyGuid': 'F000002_2', - 'numAlt': 2, 'dp': 3943, 'hl': 1.0, 'mitoCn': 214, 'contamination': 0.0, 'filters': ['artifact_prone_site'], - }}, - 'populations': { - 'seqr': {'af': 0.019480518996715546, 'ac': 3, 'an': 154}, - 'seqr_heteroplasmy': {'af': 0.006493506487458944, 'ac': 1, 'an': 154}, - 'gnomad_mito': {'af': 0.05534649267792702, 'ac': 3118, 'an': 56336}, - 'gnomad_mito_heteroplasmy': {'af': 5.3251918870955706e-05, 'ac': 3, 'an': 56336, 'max_hl': 1.0}, - 'helix': {'af': 0.04884607344865799, 'ac': 9573, 'an': 195983}, - 'helix_heteroplasmy': {'af': 9.184470400214195e-05, 'ac': 18, 'an': 195983, 'max_hl': 0.962689995765686}, - }, - 'predictions': { - 'apogee': None, - 'haplogroup_defining': 'Y', - 'hmtvar': None, - 'mitotip': None, - 'mut_taster': None, - 'sift': None, - 'mlc': 0.7514, - }, - 'commonLowHeteroplasmy': True, - 'mitomapPathogenic': True, - 'clinvar': { - 'alleleId': 150280, - 'conflictingPathogenicities': None, - 'goldStars': 0, - 'pathogenicity': 'Likely_pathogenic', - 'assertions': [], - 'version': '2024-02-21', - }, - 'transcripts': { - 'ENSG00000198727': [ - {'aminoAcids': 'L', 'canonical': 1, 'codons': 'Tta/Cta', 'geneId': 'ENSG00000198727', 'hgvsc': 'ENST00000361789.2:c.37T>C', 'hgvsp': 'ENSP00000354554.2:p.Leu13=', 'transcriptId': 'ENST00000361789', 'isLofNagnag': None, 'transcriptRank': 0, 'biotype': 'protein_coding', 'lofFilters': None, 'majorConsequence': 'synonymous_variant', 'consequenceTerms': ['synonymous_variant']}, - ], - }, - 'mainTranscriptId': 'ENST00000361789', - 'selectedMainTranscriptId': None, - '_sort': [25000014783], -} - -LOCATION_SEARCH = { - 'gene_ids': ['ENSG00000097046', 'ENSG00000177000'], - 'intervals': [['2', 1234, 5678], ['7', 1, 11100], ['1', 11785723, 11806455], ['1', 91500851, 91525764]], -} -EXCLUDE_LOCATION_SEARCH = {'intervals': LOCATION_SEARCH['intervals'], 'exclude_intervals': True} -VARIANT_ID_SEARCH = {'variant_ids': [['1', 10439, 'AC', 'A'], ['1', 91511686, 'TCA', 'G']], 'rs_ids': []} -RSID_SEARCH = {'variant_ids': [], 'rs_ids': ['rs1801131']} - -GENE_COUNTS = { - 'ENSG00000097046': {'total': 2, 'families': {'F000002_2': 2}}, - 'ENSG00000177000': {'total': 3, 'families': {'F000002_2': 2, 'F000011_11': 1}}, - 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 1, 'F000011_11': 1}}, -} - - -def get_hail_search_body(genome_version='GRCh38', num_results=100, sample_data=None, omit_data_type=None, **search_body): - sample_data = sample_data or EXPECTED_SAMPLE_DATA - if omit_data_type: - sample_data = {k: v for k, v in sample_data.items() if k != omit_data_type} - - search = { - 'sample_data': sample_data, - 'genome_version': genome_version, - 'num_results': num_results, - **search_body, - } - search.update(search_body or {}) - return search diff --git a/hail_search/web_app.py b/hail_search/web_app.py deleted file mode 100644 index 2954edc322..0000000000 --- a/hail_search/web_app.py +++ /dev/null @@ -1,138 +0,0 @@ -from aiohttp import web -import asyncio -import concurrent.futures -import ctypes -import functools -import json -import os -import hail as hl -import logging -import traceback -from typing import Callable - -from hail_search.search import search_hail_backend, load_globals, lookup_variant, lookup_variants - -logger = logging.getLogger(__name__) - -JAVA_OPTS_XSS = os.environ.get('JAVA_OPTS_XSS') -MACHINE_MEM = os.environ.get('MACHINE_MEM') -JVM_MEMORY_FRACTION = 0.9 -QUERY_TIMEOUT_S = 300 - - -def _handle_exception(e, request): - logger.error(f'{request.headers.get("From")} "{e}"') - raise e - - -@web.middleware -async def error_middleware(request, handler): - try: - return await handler(request) - except web.HTTPError as e: - _handle_exception(e, request) - except Exception as e: - error_reason = f'{e}: {traceback.format_exc()}' - _handle_exception(web.HTTPInternalServerError(reason=error_reason), request) - - -def _hl_json_default(o): - if isinstance(o, hl.Struct) or isinstance(o, hl.utils.frozendict): - return dict(o) - elif isinstance(o, set): - return sorted(o) - - -def hl_json_dumps(obj): - return json.dumps(obj, default=_hl_json_default) - -async def sync_to_async_hail_query(request: web.Request, query: Callable, *args, timeout_s=QUERY_TIMEOUT_S, **kwargs): - request_body = None - if request.body_exists: - request_body = await request.json() - - loop = asyncio.get_running_loop() - future = loop.run_in_executor(request.app.pool, functools.partial(query, request_body, *args, **kwargs)) - try: - return await asyncio.wait_for(future, timeout_s) - except asyncio.TimeoutError: - # Well documented issue with the "wait_for" approach.... the concurrent.Future is canceled but - # the underlying thread is not, allowing the Hail Query under the hood to keep running. - # https://stackoverflow.com/questions/34452590/timeout-handling-while-using-run-in-executor-and-asyncio - # This unsafe approach is taken from: - # https://stackoverflow.com/questions/323972/is-there-any-way-to-kill-a-thread - # - # A few other thoughts: - # - A ProcessPoolExecutor instead of a ThreadPoolExecutor would allow for safe worker termination - # and would potentially be a safer option in general (some portion of a hail query is cpu bound in python!) - # - A "timeout" decorator applied to the query function, catching a SIGALARM would also potentially - # suffice... but threads don't play well with signals. - # - We could also just kill the service/pod (which is fine). - for t in request.app.pool._threads: - res = ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(t.ident), ctypes.py_object(TimeoutError)) - if res > 1: - # "if it returns a number greater than one, you're in trouble, - # and you should call it again with exc=NULL to revert the effect" - ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(t.ident), None) - raise SystemExit('PyThreadState_SetAsyncExc failed') - raise TimeoutError('Hail Query Timeout Exceeded') - -async def gene_counts(request: web.Request) -> web.Response: - hail_results = await sync_to_async_hail_query(request, search_hail_backend, gene_counts=True) - return web.json_response(hail_results, dumps=hl_json_dumps) - - -async def search(request: web.Request) -> web.Response: - hail_results, total_results = await sync_to_async_hail_query(request, search_hail_backend) - return web.json_response({'results': hail_results, 'total': total_results}, dumps=hl_json_dumps) - - -async def lookup(request: web.Request) -> web.Response: - result = await sync_to_async_hail_query(request, lookup_variant) - return web.json_response(result, dumps=hl_json_dumps) - - -async def multi_lookup(request: web.Request) -> web.Response: - result = await sync_to_async_hail_query(request, lookup_variants) - return web.json_response({'results': result}, dumps=hl_json_dumps) - - -async def reload_globals(request: web.Request) -> web.Response: - result = await sync_to_async_hail_query(request, lambda _: load_globals()) - return web.json_response( - result, - dumps=hl_json_dumps - ) - - -async def status(request: web.Request) -> web.Response: - # Make sure the hail backend process is still alive. - await sync_to_async_hail_query(request, lambda _: hl.eval(1 + 1)) - return web.json_response({'success': True}) - - -async def init_web_app(): - spark_conf = {} - # memory limits adapted from https://github.com/hail-is/hail/blob/main/hail/python/hailtop/hailctl/dataproc/start.py#L321C17-L321C36 - if MACHINE_MEM: - spark_conf['spark.driver.memory'] = f'{int((int(MACHINE_MEM)-11)*JVM_MEMORY_FRACTION)}g' - if JAVA_OPTS_XSS: - spark_conf.update({f'spark.{field}.extraJavaOptions': f'-Xss{JAVA_OPTS_XSS}' for field in ['driver', 'executor']}) - hl.init(idempotent=True, spark_conf=spark_conf or None) - hl._set_flags(use_new_shuffle='1') - load_globals() - app = web.Application(middlewares=[error_middleware], client_max_size=(1024**2)*20) - app.add_routes([ - web.get('/status', status), - web.post('/reload_globals', reload_globals), - web.post('/search', search), - web.post('/gene_counts', gene_counts), - web.post('/lookup', lookup), - web.post('/multi_lookup', multi_lookup), - ]) - # The idea here is to run the hail queries off the main thread so that the - # event loop stays live and the /status check is responsive. We only - # run a single thread, though, so that hail queries block hail queries - # and we never run more than a single hail query at a time. - app.pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) - return app diff --git a/matchmaker/views/matchmaker_api.py b/matchmaker/views/matchmaker_api.py index fa0fa3cf15..f3a7dfffe4 100644 --- a/matchmaker/views/matchmaker_api.py +++ b/matchmaker/views/matchmaker_api.py @@ -2,8 +2,10 @@ import requests from datetime import datetime from django.core.mail.message import EmailMessage -from django.db.models import prefetch_related_objects, Q +from django.db.models import prefetch_related_objects, Q, F, Value, CharField +from django.db.models.functions import Coalesce +from clickhouse_search.search import get_transcripts_by_key from matchmaker.models import MatchmakerResult, MatchmakerContactNotes, MatchmakerSubmission, MatchmakerSubmissionGenes, \ MatchmakerIncomingQuery from matchmaker.matchmaker_utils import get_mme_genes_phenotypes_for_results, parse_mme_patient, \ @@ -13,6 +15,8 @@ from seqr.utils.communication_utils import safe_post_to_slack from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException +from seqr.utils.search.utils import backend_specific_call +from seqr.utils.xpos_utils import get_chrom_pos from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json from seqr.views.utils.json_utils import create_json_response @@ -47,10 +51,21 @@ def get_individual_mme_matches(request, submission_guid): results = MatchmakerResult.objects.filter(submission=submission) response_json = get_json_for_saved_variants_with_tags( - SavedVariant.objects.filter(family=submission.individual.family), add_details=True, genome_version=project.genome_version) + SavedVariant.objects.filter(family=submission.individual.family), genome_version=project.genome_version, additional_values={ + 'genomeVersion': Coalesce('saved_variant_json__genomeVersion', Value(project.genome_version), output_field=CharField()), + 'transcripts': F('saved_variant_json__transcripts'), + }, additional_model_fields=backend_specific_call([], ['key', 'genotypes']), + ) + + variants = response_json['savedVariantsByGuid'].values() + backend_specific_call(lambda *args: None, _add_clickhouse_transcripts)( + variants, project.genome_version, request.user, + ) gene_ids = set() - for variant in response_json['savedVariantsByGuid'].values(): + for variant in variants: + chrom, pos = get_chrom_pos(variant['xpos']) + variant.update({'chrom': chrom, 'pos': pos}) gene_ids.update(list(variant.get('transcripts', {}).keys())) hpo_terms_by_id = get_hpo_terms_by_id( @@ -62,6 +77,17 @@ def get_individual_mme_matches(request, submission_guid): submission, results, request.user, additional_genes=gene_ids, response_json=response_json) +def _add_clickhouse_transcripts(variants, genome_version, user): + keys = {variant['key'] for variant in variants} + try: + transcripts_by_key = get_transcripts_by_key(genome_version, keys) + except Exception as e: + logger.error(f'Error loading transcripts from clickhouse: {e}', user) + transcripts_by_key = {} + for variant in variants: + variant['transcripts'] = transcripts_by_key.get(variant['key'], {}) + + @login_and_policies_required def get_mme_nodes(request): return create_json_response({'mmeNodes': list(MME_NODES_BY_NAME.keys())}) diff --git a/matchmaker/views/matchmaker_api_tests.py b/matchmaker/views/matchmaker_api_tests.py index bd7e123046..bea9a7694c 100644 --- a/matchmaker/views/matchmaker_api_tests.py +++ b/matchmaker/views/matchmaker_api_tests.py @@ -12,7 +12,7 @@ update_mme_submission, delete_mme_submission, update_mme_result_status, send_mme_contact_email, \ get_mme_nodes, search_local_individual_mme_matches, finalize_mme_search, \ update_mme_contact_note, update_mme_project_contact -from seqr.views.utils.test_utils import AuthenticationTestCase +from seqr.views.utils.test_utils import AuthenticationTestCase, SAVED_VARIANT_FIELDS INDIVIDUAL_GUID = 'I000001_na19675' SUBMISSION_GUID = 'MS000001_na19675' @@ -230,6 +230,8 @@ def test_get_individual_mme_matches(self): set(response_json['savedVariantsByGuid'].keys()), {'SV0000001_2103343353_r0390_100', 'SV0059957_11562437_f019313_1', 'SV0059956_11560662_f019313_1'} ) + saved_variant_fields = {*SAVED_VARIANT_FIELDS, 'chrom', 'pos', 'transcripts', 'genomeVersion', 'noteGuids', 'functionalDataGuids', 'tagGuids'} + self.assertSetEqual(set(response_json['savedVariantsByGuid']['SV0000001_2103343353_r0390_100'].keys()), saved_variant_fields) self.assertDictEqual(response_json['mmeContactNotes'], {}) # users should see originating query for results if the have correct project permissions diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 40033f03c7..27d38ed371 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -8,14 +8,11 @@ import re from clickhouse_search.search import get_clickhouse_genotypes -from reference_data.models import GENOME_VERSION_LOOKUP, GENOME_VERSION_GRCh38 +from reference_data.models import GENOME_VERSION_LOOKUP from seqr.models import Family, Sample, SavedVariant, Project, Individual from seqr.utils.communication_utils import safe_post_to_slack, send_project_email from seqr.utils.file_utils import file_iter, list_files, is_google_bucket_file_path from seqr.utils.search.add_data_utils import notify_search_data_loaded, update_airtable_loading_tracking_status -from seqr.utils.search.utils import parse_valid_variant_id, backend_specific_call -from seqr.utils.search.hail_search_utils import hail_variant_multi_lookup, search_data_type -from seqr.utils.xpos_utils import get_xpos, CHROMOSOMES, MIN_POS, MAX_POS from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS from seqr.views.utils.dataset_utils import match_and_update_search_samples from seqr.views.utils.export_utils import write_multiple_files @@ -29,7 +26,6 @@ CLICKHOUSE_MIGRATION_SENTINEL = 'hail_search_to_clickhouse_migration' RUN_FILE_PATH_TEMPLATE = '{data_dir}/{genome_version}/{dataset_type}/runs/{run_version}/{file_name}' -SUCCESS_FILE_NAME = '_SUCCESS' CLICKHOUSE_SUCCESS_FILE_NAME = '_CLICKHOUSE_LOAD_SUCCESS' VALIDATION_ERRORS_FILE_NAME = 'validation_errors.json' ERRORS_REPORTED_FILE_NAME = '_ERRORS_REPORTED' @@ -41,8 +37,6 @@ Sample.DATASET_TYPE_SV_CALLS: f'{Sample.DATASET_TYPE_SV_CALLS}_WGS', } USER_EMAIL = 'manage_command' -MAX_LOOKUP_VARIANTS = 1000 -MAX_RELOAD_VARIANTS = 100000 RELATEDNESS_CHECK_NAME = 'relatedness_check' PDO_COPY_FIELDS = [ @@ -69,8 +63,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): runs = self._get_runs(**options) - success_file_name = backend_specific_call(None, SUCCESS_FILE_NAME, CLICKHOUSE_SUCCESS_FILE_NAME) - success_run_dirs = [run_dir for run_dir, run_details in runs.items() if success_file_name in run_details['files']] + success_run_dirs = [run_dir for run_dir, run_details in runs.items() if CLICKHOUSE_SUCCESS_FILE_NAME in run_details['files']] if success_run_dirs: self._load_success_runs(runs, success_run_dirs) if not success_run_dirs: @@ -95,38 +88,19 @@ def _load_success_runs(self, runs, success_run_dirs): return logger.info(f'Loading new samples from {len(success_run_dirs)} run(s)') - updated_families_by_data_type = defaultdict(set) - updated_variants_by_data_type = defaultdict(dict) for run_dir, run_details in new_runs.items(): try: if CLICKHOUSE_MIGRATION_SENTINEL in run_details["run_version"]: logging.info(f'Skipping ClickHouse migration {run_details["genome_version"]}/{run_details["dataset_type"]}: {run_details["run_version"]}') continue metadata_path = os.path.join(run_dir, 'metadata.json') - data_type, updated_families, updated_variants_by_id = self._load_new_samples(metadata_path, **run_details) - data_type_key = (data_type, run_details['genome_version']) - updated_families_by_data_type[data_type_key].update(updated_families) - updated_variants_by_data_type[data_type_key].update(updated_variants_by_id) + self._load_new_samples(metadata_path, **run_details) except Exception as e: logger.error(f'Error loading {run_details["run_version"]}: {e}') # Reset cached results for all projects, as seqr AFs will have changed for all projects when new data is added reset_cached_search_results(project=None) - backend_specific_call( - lambda *args: None, self._reload_dataset_type_shared_variant_annotations, lambda *args: None, - )(updated_families_by_data_type, updated_variants_by_data_type) - - @classmethod - def _reload_dataset_type_shared_variant_annotations(cls, updated_families_by_data_type, updated_variants_by_data_type): - for data_type_key, updated_families in updated_families_by_data_type.items(): - try: - cls._reload_shared_variant_annotations( - *data_type_key, updated_variants_by_data_type[data_type_key], exclude_families=updated_families, - ) - except Exception as e: - logger.error(f'Error reloading shared annotations for {"/".join(data_type_key)}: {e}') - @classmethod def _get_runs(cls, **kwargs): path = cls._run_path(lambda field: kwargs.get(field, '*') or '*') @@ -279,12 +253,9 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers logger.error(f'Error updating individuals sample qc {run_version}: {e}') # Reload saved variant JSON - update_function = backend_specific_call(None, None, cls._update_project_saved_variant_genotypes) - updated_variants_by_id = update_projects_saved_variant_json([ + update_projects_saved_variant_json([ (project.id, project.guid, project.name, project.genome_version, families) for project, families in families_by_project.items() - ], user_email=USER_EMAIL, dataset_type=dataset_type, update_function=update_function, samples=updated_samples, clickhouse_dataset_type=clickhouse_dataset_type) - - return search_data_type(dataset_type, sample_type), set(family_project_map.keys()), updated_variants_by_id + ], user_email=USER_EMAIL, dataset_type=dataset_type, update_function=cls._update_project_saved_variant_genotypes, samples=updated_samples, clickhouse_dataset_type=clickhouse_dataset_type) @classmethod def _is_internal_project(cls, project): @@ -440,96 +411,6 @@ def _update_project_saved_variant_genotypes(cls, project_id, genome_version, use updates.update({v.id: v for v in variants}) return updates - @classmethod - def _reload_shared_variant_annotations(cls, data_type, genome_version, updated_variants_by_id=None, exclude_families=None, chromosomes=None): - dataset_type = data_type.split('_')[0] - is_sv = dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS) - dataset_type = data_type.split('_')[0] if is_sv else data_type - db_genome_version = genome_version.replace('GRCh', '') - updated_annotation_samples = Sample.objects.filter( - is_active=True, dataset_type=dataset_type, - individual__family__project__genome_version=db_genome_version, - ) - if exclude_families: - updated_annotation_samples = updated_annotation_samples.exclude(individual__family__guid__in=exclude_families) - if is_sv: - updated_annotation_samples = updated_annotation_samples.filter(sample_type=data_type.split('_')[1]) - - variant_models = get_saved_variants( - genome_version, dataset_type=dataset_type, - family_guids=updated_annotation_samples.values_list('individual__family__guid', flat=True).distinct(), - ) - - variant_type_summary = f'{data_type} {genome_version} saved variants' - - if updated_variants_by_id: - fetched_variant_models = variant_models.filter(variant_id__in=updated_variants_by_id.keys()) - if fetched_variant_models: - logger.info(f'Reloading shared annotations for {len(fetched_variant_models)} fetched {variant_type_summary}') - for variant_model in fetched_variant_models: - updated_variant = { - k: v for k, v in updated_variants_by_id[variant_model.variant_id].items() - if k not in {'familyGuids', 'genotypes'} - } - variant_model.saved_variant_json.update(updated_variant) - SavedVariant.bulk_update_models(None, fetched_variant_models, ['saved_variant_json']) - - variant_models = variant_models.exclude(variant_id__in=updated_variants_by_id.keys()) - - chromosomes = cls._get_chroms_to_reload(chromosomes, dataset_type, genome_version, variant_models) - if chromosomes is None: - return - - for chrom in chromosomes: - cls._reload_shared_variant_annotations_by_chrom(chrom, variant_models, data_type, genome_version, variant_type_summary) - - @staticmethod - def _get_chroms_to_reload(chromosomes, dataset_type, genome_version, variant_models): - if dataset_type != Sample.DATASET_TYPE_VARIANT_CALLS: - return [None] - if chromosomes: - return chromosomes - - num_reload = variant_models.count() - if genome_version == GENOME_VERSION_LOOKUP[GENOME_VERSION_GRCh38] and num_reload > MAX_RELOAD_VARIANTS: - logger.info(f'Skipped reloading all {num_reload} saved variant annotations for {dataset_type} {genome_version}') - return None - - return CHROMOSOMES - - @classmethod - def _reload_shared_variant_annotations_by_chrom(cls, chrom, variant_models, data_type, genome_version, variant_type_summary): - if chrom: - variant_models = variant_models.filter(xpos__gte=get_xpos(chrom, MIN_POS), xpos__lte=get_xpos(chrom, MAX_POS)) - - chrom_summary = f' in chromosome {chrom}' if chrom else '' - if not variant_models: - logger.info(f'No additional {variant_type_summary} to update{chrom_summary}') - return - - variants_by_id = defaultdict(list) - for v in variant_models: - variants_by_id[v.variant_id].append(v) - - logger.info(f'Reloading shared annotations for {len(variant_models)} {variant_type_summary}{chrom_summary} ({len(variants_by_id)} unique)') - - variant_ids = sorted(variants_by_id.keys()) - if chrom: - variant_ids = sorted([parse_valid_variant_id(variant_id) for variant_id in variant_ids]) - - for i in range(0, len(variant_ids), MAX_LOOKUP_VARIANTS): - updated_variants = hail_variant_multi_lookup(USER_EMAIL, variant_ids[i:i+MAX_LOOKUP_VARIANTS], data_type, genome_version) - logger.info(f'Fetched {len(updated_variants)} additional variants{chrom_summary}') - - updated_variant_models = [] - for variant in updated_variants: - for variant_model in variants_by_id[variant['variantId']]: - variant_model.saved_variant_json.update(variant) - updated_variant_models.append(variant_model) - - SavedVariant.bulk_update_models(None, updated_variant_models, ['saved_variant_json']) - -reload_shared_variant_annotations = Command._reload_shared_variant_annotations update_individuals_sample_qc = Command._update_individuals_sample_qc get_pipeline_runs = Command._get_runs diff --git a/seqr/management/commands/deactivate_project_search.py b/seqr/management/commands/deactivate_project_search.py index 4c1f0c6cff..8d8fd2f417 100644 --- a/seqr/management/commands/deactivate_project_search.py +++ b/seqr/management/commands/deactivate_project_search.py @@ -26,7 +26,7 @@ def handle(self, *args, **options): dataset_types = Sample.objects.filter(guid__in=updated).values_list('dataset_type', 'sample_type').order_by('dataset_type').distinct() for dataset_type, sample_type in dataset_types: backend_specific_call( - lambda *args, **kwargs: True, lambda *args, **kwargs: True, self._delete_clickhouse_project, + lambda *args, **kwargs: True, self._delete_clickhouse_project, )(project, dataset_type, sample_type) @staticmethod diff --git a/seqr/management/commands/reload_saved_variant_annotations.py b/seqr/management/commands/reload_saved_variant_annotations.py deleted file mode 100644 index a2481e1731..0000000000 --- a/seqr/management/commands/reload_saved_variant_annotations.py +++ /dev/null @@ -1,27 +0,0 @@ -from django.core.management.base import BaseCommand, CommandError -from reference_data.models import GENOME_VERSION_LOOKUP -from seqr.models import Sample -from seqr.management.commands.check_for_new_samples_from_pipeline import reload_shared_variant_annotations -from seqr.utils.search.hail_search_utils import search_data_type -from seqr.utils.search.utils import backend_specific_call - -DATA_TYPE_CHOICES = { - search_data_type(dt, st) for dt in Sample.DATASET_TYPE_LOOKUP for st in [Sample.SAMPLE_TYPE_WGS, Sample.SAMPLE_TYPE_WES] -} - - -def _clickhouse_error(): - raise CommandError('Reloading variant annotations is not supported in clickhouse') - - -class Command(BaseCommand): - help = 'Reload shared variant annotations for all saved variants' - - def add_arguments(self, parser): - parser.add_argument('data_type', choices=sorted(DATA_TYPE_CHOICES)) - parser.add_argument('genome_version', choices=sorted(GENOME_VERSION_LOOKUP.values())) - parser.add_argument('chromosomes', nargs='*', help='Chromosome(s) to reload. If not specified, defaults to all chromosomes.') - - def handle(self, *args, **options): - backend_specific_call(lambda: True, lambda: True, _clickhouse_error)() - reload_shared_variant_annotations(options['data_type'], options['genome_version'], chromosomes=options['chromosomes']) diff --git a/seqr/management/commands/reload_saved_variant_json.py b/seqr/management/commands/reload_saved_variant_json.py index 1f76eb2fa4..6d64f991e7 100644 --- a/seqr/management/commands/reload_saved_variant_json.py +++ b/seqr/management/commands/reload_saved_variant_json.py @@ -2,16 +2,13 @@ from django.core.management.base import BaseCommand, CommandError from django.db.models.query_utils import Q from seqr.models import Project -from seqr.utils.search.utils import backend_specific_call +from seqr.utils.search.elasticsearch.es_utils import update_project_saved_variant_json +from seqr.utils.search.utils import es_only from seqr.views.utils.variant_utils import update_projects_saved_variant_json logger = logging.getLogger(__name__) -def _clickhouse_error(): - raise CommandError('Reloading variants is not supported in clickhouse') - - class Command(BaseCommand): help = 'Transfer projects to the new seqr schema' @@ -19,8 +16,8 @@ def add_arguments(self, parser): parser.add_argument('projects', nargs="*", help='Project(s) to transfer. If not specified, defaults to all projects.') parser.add_argument('--family-guid', help='optional family to reload variants for') + @es_only def handle(self, *args, **options): - backend_specific_call(lambda: True, lambda: True, _clickhouse_error)() projects_to_process = options['projects'] family_guid = options['family_guid'] @@ -33,5 +30,5 @@ def handle(self, *args, **options): family_ids = [family_guid] if family_guid else None project_list = [(*project, family_ids) for project in projects.order_by('id').values_list('id', 'guid', 'name', 'genome_version')] - update_projects_saved_variant_json(project_list, user_email='manage_command') + update_projects_saved_variant_json(project_list, user_email='manage_command', update_function=update_project_saved_variant_json) logger.info("Done") diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index 7744cb905d..9f66ae414d 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -10,7 +10,6 @@ def _disable_search(families, from_project): search_samples = Sample.objects.filter(is_active=True, individual__family__in=families) - updated_family_dataset_types = None if search_samples: updated_families = search_samples.values_list("individual__family__family_id", flat=True).distinct() updated_family_dataset_types = list(search_samples.values_list('individual__family__guid', 'dataset_type', 'sample_type').distinct()) @@ -19,12 +18,8 @@ def _disable_search(families, from_project): logger.info( f'Disabled search for {num_updated} samples in the following {len(updated_families)} families: {family_summary}' ) - return updated_family_dataset_types - -def _disable_search_clickhouse(families, from_project): - updated_family_dataset_types = _disable_search(families, from_project) - for update in updated_family_dataset_types: - logger.info(delete_clickhouse_family(from_project, *update)) + for update in updated_family_dataset_types: + logger.info(delete_clickhouse_family(from_project, *update)) class Command(BaseCommand): @@ -54,7 +49,7 @@ def handle(self, *args, **options): ] logger.info(f'Skipping {num_found - len(families)} families with analysis groups in the project: {", ".join(group_families)}') - backend_specific_call(lambda *args: None, _disable_search, _disable_search_clickhouse)(families, from_project) + backend_specific_call(lambda *args: None, _disable_search)(families, from_project) for variant_tag_type in VariantTagType.objects.filter(project=from_project): variant_tags = VariantTag.objects.filter(saved_variants__family__in=families, variant_tag_type=variant_tag_type) diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 0700851413..c3ba7986c9 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -6,14 +6,12 @@ import mock import responses -from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, AuthenticationTestCase +from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, AuthenticationTestCase, DifferentDbTransactionSupportMixin from seqr.models import Project, Family, Individual, Sample, SavedVariant SEQR_URL = 'https://seqr.broadinstitute.org/' PROJECT_GUID = 'R0003_test' EXTERNAL_PROJECT_GUID = 'R0004_non_analyst_project' -MOCK_HAIL_HOST = 'test-hail-host' -MOCK_HAIL_ORIGIN = f'http://{MOCK_HAIL_HOST}' GUID_ID = 54321 GCNV_GUID_ID = 12345 @@ -161,12 +159,12 @@ '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/manual__2025-01-13/_ERRORS_REPORTED', '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/manual__2025-01-13/validation_errors.json', '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/manual__2025-01-14/validation_errors.json', - '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/auto__2023-08-09/_SUCCESS', - '/seqr/seqr-hail-search-data/GRCh37/SNV_INDEL/runs/manual__2023-11-02/_SUCCESS', - '/seqr/seqr-hail-search-data/GRCh38/MITO/runs/auto__2024-08-12/_SUCCESS', - '/seqr/seqr-hail-search-data/GRCh38/GCNV/runs/auto__2024-09-14/_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/auto__2023-08-09/_CLICKHOUSE_LOAD_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh37/SNV_INDEL/runs/manual__2023-11-02/_CLICKHOUSE_LOAD_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh38/MITO/runs/auto__2024-08-12/_CLICKHOUSE_LOAD_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh38/GCNV/runs/auto__2024-09-14/_CLICKHOUSE_LOAD_SUCCESS', '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/manual__2025-01-24/validation_errors.json', - '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/hail_search_to_clickhouse_migration_WGS_R0877_neptune/_SUCCESS', + '/seqr/seqr-hail-search-data/GRCh38/SNV_INDEL/runs/hail_search_to_clickhouse_migration_WGS_R0877_neptune/_CLICKHOUSE_LOAD_SUCCESS', ] RUN_PATHS = [ b'gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-13/', @@ -375,10 +373,9 @@ def set_up(self): self.addCleanup(patcher.stop) Sample.objects.filter(guid=OLD_DATA_SAMPLE_GUID).update(sample_type='WES') - def _test_call(self, error_logs=None, reload_annotations_logs=None, run_loading_logs=None, reload_calls=None, num_runs=5, has_reload_calls=False): + def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=5): self._set_loading_files() self.reset_logs() - responses.calls.reset() call_command('check_for_new_samples_from_pipeline') @@ -407,51 +404,23 @@ def _test_call(self, error_logs=None, reload_annotations_logs=None, run_loading_ {'severity': 'ERROR', '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent'}, )) logs.append(('Reset 2 cached results', None)) - logs += reload_annotations_logs or [] logs += [] if single_call else [(log, None) for log in self.VALIDATION_LOGS] logs.append(('DONE', None)) self.assert_json_logs(user=None, expected=logs) self.mock_redis.return_value.delete.assert_called_with('search_results__*', 'variant_lookup_results__*') - # Test reload saved variants - num_airtable_loading_calls, num_airtable_validation_calls = self._assert_expected_airtable_calls(has_reload_calls, single_call) - if not reload_calls: - self.assertEqual(len(responses.calls), num_airtable_loading_calls + num_airtable_validation_calls) - return - - reload_annotation_calls = [] if single_call else [ - {'genome_version': 'GRCh38', 'data_type': 'SNV_INDEL', 'variant_ids': [['1', 1562437, 'G', 'CA']]}, - {'genome_version': 'GRCh38', 'data_type': 'SNV_INDEL', 'variant_ids': [['1', 1562437, 'G', 'CA']]}, - {'genome_version': 'GRCh38', 'data_type': 'SNV_INDEL', 'variant_ids': [['1', 46859832, 'G', 'A']]}, - {'genome_version': 'GRCh38', 'data_type': 'SV_WES', 'variant_ids': ['prefix_19107_DEL']} - ] - self.assertEqual(len(responses.calls), len(reload_calls) +len(reload_annotation_calls) + num_airtable_loading_calls + num_airtable_validation_calls) - for i, call in enumerate(reload_calls or []): - resp = responses.calls[i+num_airtable_loading_calls] - self.assertEqual(resp.request.url, f'{MOCK_HAIL_ORIGIN}:5000/search') - self.assertEqual(resp.request.headers.get('From'), 'manage_command') - self.assertDictEqual(json.loads(resp.request.body), call) - - for i, body in enumerate(reload_annotation_calls): - multi_lookup_request = responses.calls[num_airtable_loading_calls+len(reload_calls)+i].request - self.assertEqual(multi_lookup_request.url, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup') - self.assertEqual(multi_lookup_request.headers.get('From'), 'manage_command') - self.assertDictEqual(json.loads(multi_lookup_request.body), body) + num_calls = self._assert_expected_airtable_calls(bool(run_loading_logs), single_call) + self.assertEqual(len(responses.calls), num_calls) def _additional_loading_logs(self, data_type, version): return [] - @mock.patch('seqr.management.commands.check_for_new_samples_from_pipeline.MAX_LOOKUP_VARIANTS', 1) @mock.patch('seqr.views.utils.airtable_utils.BASE_URL', 'https://test-seqr.org/') @mock.patch('seqr.views.utils.airtable_utils.MAX_UPDATE_RECORDS', 2) - @mock.patch('seqr.management.commands.check_for_new_samples_from_pipeline.MAX_RELOAD_VARIANTS') @mock.patch('seqr.views.utils.export_utils.TemporaryDirectory') @mock.patch('seqr.utils.communication_utils.EmailMultiAlternatives') - @responses.activate - def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): - mock_max_reload_variants.__lt__.return_value = False - + def test_command(self, mock_email, mock_temp_dir): # Test errors self._set_empty_loading_files() with self.assertRaises(CommandError) as ce: @@ -485,69 +454,33 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): # Test success self.mock_send_slack.reset_mock() mock_email.reset_mock() - search_body = { - 'genome_version': 'GRCh38', 'num_results': 1, 'variant_ids': [['1', 248367227, 'TC', 'T']], 'variant_keys': [], - } - reload_snv_indel_calls = [ - {**search_body, 'sample_data': {'SNV_INDEL': [ - {'individual_guid': 'I000016_na20888', 'family_guid': 'F000012_12', 'project_guid': 'R0003_test', 'affected': 'A', 'sample_id': 'NA20888', 'sample_type': 'WES'}, - {'individual_guid': 'I000017_na20889', 'family_guid': 'F000012_12', 'project_guid': 'R0003_test', 'affected': 'A', 'sample_id': 'NA20889', 'sample_type': 'WES'}, - ]}}, - {**search_body, 'sample_data': {'SNV_INDEL': [ - {'individual_guid': 'I000018_na21234', 'family_guid': 'F000014_14', 'project_guid': 'R0004_non_analyst_project', 'affected': 'A', 'sample_id': 'NA21234', 'sample_type': 'WES'}, - ]}}, - ] - reload_fetched_annotations_logs = [ - ('Reloading shared annotations for 1 fetched SNV_INDEL GRCh38 saved variants', None), - ('update 1 SavedVariants', {'dbUpdate': { - 'dbEntity': 'SavedVariant', - 'entityIds': ['SV0000002_1248367227_r0390_100'], - 'updateFields': ['saved_variant_json'], - 'updateType': 'bulk_update'}, - }), - ] create_snv_indel_samples_logs = [ ('Loading 4 WES SNV_INDEL samples in 2 projects', None), ('create 4 Samples', {'dbUpdate': mock.ANY}), ('update 4 Samples', {'dbUpdate': mock.ANY}), ] - update_sample_qc_logs = [ + update_sample_logs = [ ('update 2 Individuals', {'dbUpdate': { 'dbEntity': 'Individual', 'entityIds': ['I000001_na19675', 'I000015_na20885'], 'updateFields': ['filter_flags', 'pop_platform_filters', 'population'], 'updateType': 'bulk_update'}} - ), + ), + ('Reloading saved variants in 2 projects', None), + ('Reloading genotypes for 0 SNV_INDEL variants in family F000012_12', None), + ('Updated 0 variants in 2 families for project Test Reprocessed Project', None), + ('Reloading genotypes for 1 SNV_INDEL variants in family F000014_14', None), + ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), + ('Updated 1 variants in 1 families for project Non-Analyst Project', None), + ('Reload Summary: ', None), + (' Non-Analyst Project: Updated 1 variants', None), ] - self._test_call(reload_calls=reload_snv_indel_calls + [ - {'genome_version': 'GRCh38', 'num_results': 1, 'variant_ids': [], 'variant_keys': ['prefix_19107_DEL'], 'sample_data': {'SV_WES': [ - {'individual_guid': 'I000017_na20889', 'family_guid': 'F000012_12', 'project_guid': 'R0003_test', 'affected': 'A', 'sample_id': 'NA20889', 'sample_type': 'WES'}, - ]}}, - ], reload_annotations_logs=reload_fetched_annotations_logs + [ - ('Reloading shared annotations for 2 SNV_INDEL GRCh38 saved variants in chromosome 1 (2 unique)', None), - ('Fetched 1 additional variants in chromosome 1', None), - ('update 1 SavedVariants', {'dbUpdate': { - 'dbEntity': 'SavedVariant', - 'entityIds': ['SV0059956_11560662_f019313_1'], - 'updateFields': ['saved_variant_json'], - 'updateType': 'bulk_update'}, - }), - ('Fetched 1 additional variants in chromosome 1', None), - ('update 1 SavedVariants', {'dbUpdate': { - 'dbEntity': 'SavedVariant', - 'entityIds': ['SV0059956_11560662_f019313_1'], - 'updateFields': ['saved_variant_json'], - 'updateType': 'bulk_update'}, - }), - ] + [(f'No additional SNV_INDEL GRCh38 saved variants to update in chromosome {chrom}', None) for chrom in ['2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', 'X', 'Y', 'M']] + [ - ('Reloading shared annotations for 1 SV_WES GRCh38 saved variants (1 unique)', None), - ('Fetched 1 additional variants', None), - ], run_loading_logs={ + self._test_call(run_loading_logs={ 'GRCh38/SNV_INDEL': create_snv_indel_samples_logs + [ ('update 1 Samples', {'dbUpdate': mock.ANY}), ('update 2 Familys', {'dbUpdate': mock.ANY}), ] + self.AIRTABLE_LOGS + [ ('update 3 Familys', {'dbUpdate': mock.ANY}), - ] + update_sample_qc_logs + self.RELOAD_PROJECT_VARIANT_LOGS, + ] + update_sample_logs, 'GRCh38/MITO': [ ('Loading 2 WGS MITO samples in 1 projects', None) ], @@ -558,7 +491,10 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): ('update 3 Samples', {'dbUpdate': mock.ANY}), ('update 1 Familys', {'dbUpdate': mock.ANY}), ('Reloading saved variants in 2 projects', None), - ] + self.RELOAD_SV_VARIANT_LOGS, + ('Updated 0 variants in 1 families for project 1kg project nรฅme with uniรงรธde', None), + ('Updated 0 variants in 1 families for project Test Reprocessed Project', None), + ('Reload Summary: ', None), + ], }, error_logs={ 'manual__2023-11-02': 'Invalid families in run metadata GRCh37/SNV_INDEL: manual__2023-11-02 - F0000123_ABC', 'auto__2024-08-12': 'Matches not found for sample ids: NA20885, NA22882', @@ -650,7 +586,11 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): ) self.assertEqual(Family.objects.get(guid='F000014_14').analysis_status, 'Rncc') - self._assert_saved_variant_models_updated() + saved_variant = SavedVariant.objects.get(key=100, family_id=14) + self.assertDictEqual(saved_variant.genotypes, {'I000018_na21234': { + 'ab': 0.0, 'dp': 49, 'gq': 99, 'numAlt': 2, 'filters': [], + 'sampleId': 'NA21234', 'familyGuid': 'F000014_14', 'sampleType': 'WGS', 'individualGuid': 'I000018_na21234', + }}) # Test notifications self.assertEqual(self.mock_send_slack.call_count, 7 + len(self.ADDITIONAL_SLACK_CALLS)) @@ -741,20 +681,18 @@ def test_command(self, mock_email, mock_temp_dir, mock_max_reload_variants): self.mock_redis.return_value.delete.assert_not_called() # Test reloading shared annotations is skipped if too many saved variants - mock_max_reload_variants.__lt__.return_value = True snv_indel_samples.delete() airtable_logs = self.AIRTABLE_LOGS[:-1] if self.AIRTABLE_LOGS: airtable_logs.append(('Fetched 1 AnVIL Seqr Loading Requests Tracking records from airtable', None)) - self._test_call(num_runs=2, reload_annotations_logs=reload_fetched_annotations_logs + [ - ('Skipped reloading all 2 saved variant annotations for SNV_INDEL GRCh38', None), - ], reload_calls=reload_snv_indel_calls, run_loading_logs={ - 'GRCh38/SNV_INDEL': create_snv_indel_samples_logs + airtable_logs + update_sample_qc_logs + self.RELOAD_PROJECT_VARIANT_LOGS, + self._test_call(num_runs=2, run_loading_logs={ + 'GRCh38/SNV_INDEL': create_snv_indel_samples_logs + airtable_logs + update_sample_logs, }) -@mock.patch('seqr.utils.search.hail_search_utils.HAIL_BACKEND_SERVICE_HOSTNAME', MOCK_HAIL_HOST) -class LocalCheckNewSamplesTest(AuthenticationTestCase, CheckNewSamplesTest): - fixtures = ['users', '1kg_project'] + +class LocalCheckNewSamplesTest(DifferentDbTransactionSupportMixin, AuthenticationTestCase, CheckNewSamplesTest): + fixtures = ['users', '1kg_project', 'clickhouse_saved_variants'] + databases = '__all__' ES_HOSTNAME = '' @@ -766,22 +704,6 @@ class LocalCheckNewSamplesTest(AuthenticationTestCase, CheckNewSamplesTest): LIST_FILE_LOGS = [] AIRTABLE_LOGS = [] VALIDATION_LOGS = [] - RELOAD_PROJECT_VARIANT_LOGS = [ - ('Reloading saved variants in 2 projects', None), - ('Updated 0 variants in 2 families for project Test Reprocessed Project', None), - ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), - ('Updated 1 variants in 1 families for project Non-Analyst Project', None), - ('Reload Summary: ', None), - (' Non-Analyst Project: Updated 1 variants', None), - ] - RELOAD_SV_VARIANT_LOGS = [ - (mock.ANY, {'severity': 'ERROR', '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent'}), - ('Error reloading variants in Test Reprocessed Project: Bad Request', {'severity': 'ERROR','@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent'}), - ('Reload Summary: ', None), - ('Skipped the following 1 project with no saved variants: 1kg project nรฅme with uniรงรธde', None), - ('1 failed projects', None), - (' Test Reprocessed Project: Bad Request', None), - ] ADDITIONAL_SLACK_CALLS = [ mock.call( 'seqr-data-loading', @@ -809,26 +731,6 @@ def setUp(self): self.set_up() super().setUp() - def test_command(self, *args, **kwargs): - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=200, json={ - 'results': [{'variantId': '1-248367227-TC-T', 'familyGuids': ['F000014_14'], 'updated_field': 'updated_value'}], - 'total': 1, - }) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup', status=400) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup', status=200, json={ - 'results': [{'variantId': '1-46859832-G-A', 'updated_new_field': 'updated_value', 'rsid': 'rs123'}], - }) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=200, json={ - 'results': [{'variantId': '1-248367227-TC-T', 'familyGuids': ['F000014_14'], 'updated_field': 'updated_value'}], - 'total': 1, - }) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=400) - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/search', status=200, json={ - 'results': [{'variantId': '1-248367227-TC-T', 'familyGuids': ['F000014_14'], 'updated_field': 'updated_value'}], - 'total': 1, - }) - super().test_command(*args, **kwargs) - def _set_empty_loading_files(self): self.mock_glob.return_value = [] @@ -850,7 +752,7 @@ def _assert_expected_loading_file_calls(self, single_call): self.mock_glob.assert_called_with('/seqr/seqr-hail-search-data/*/*/runs/*/*', recursive=False) self.mock_open.assert_has_calls([ mock.call(LOCAL_RUN_PATHS[2], 'r'), - *[mock.call(path.replace('_SUCCESS', 'metadata.json'), 'r') for path in LOCAL_RUN_PATHS[3:-1]] + *[mock.call(path.replace('_CLICKHOUSE_LOAD_SUCCESS', 'metadata.json'), 'r') for path in LOCAL_RUN_PATHS[3:-1]] ], any_order=True) self.assertEqual(self.mock_mkdir.call_count, 0 if single_call else 2) self.assertEqual(list(self.mock_written_files.keys()), [ @@ -859,28 +761,7 @@ def _assert_expected_loading_file_calls(self, single_call): ]) def _assert_expected_airtable_calls(self, *args, **kwargs): - return 0, 0 - - def _assert_saved_variant_models_updated(self): - updated_variants = SavedVariant.objects.filter(saved_variant_json__updated_field='updated_value') - self.assertEqual(len(updated_variants), 2) - self.assertSetEqual( - {v.guid for v in updated_variants}, - {'SV0000006_1248367227_r0004_non', 'SV0000002_1248367227_r0390_100'} - ) - reloaded_variant = next(v for v in updated_variants if v.guid == 'SV0000006_1248367227_r0004_non') - annotation_updated_variant = next(v for v in updated_variants if v.guid == 'SV0000002_1248367227_r0390_100') - self.assertEqual(len(reloaded_variant.saved_variant_json), 3) - self.assertListEqual(reloaded_variant.saved_variant_json['familyGuids'], ['F000014_14']) - self.assertEqual(len(annotation_updated_variant.saved_variant_json), 17) - self.assertListEqual(annotation_updated_variant.saved_variant_json['familyGuids'], ['F000001_1']) - - annotation_updated_json = SavedVariant.objects.get(guid='SV0059956_11560662_f019313_1').saved_variant_json - self.assertEqual(len(annotation_updated_json), 16) - self.assertEqual(annotation_updated_json['updated_new_field'], 'updated_value') - self.assertEqual(annotation_updated_json['rsid'], 'rs123') - self.assertEqual(annotation_updated_json['mainTranscriptId'], 'ENST00000505820') - self.assertEqual(len(annotation_updated_json['genotypes']), 3) + return 0 class AirtableCheckNewSamplesTest(AnvilAuthenticationTestCase, CheckNewSamplesTest): @@ -925,21 +806,6 @@ class AirtableCheckNewSamplesTest(AnvilAuthenticationTestCase, CheckNewSamplesTe '==> gsutil cat gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-24/validation_errors.json', '==> gsutil mv /mock/tmp/* gs://seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2025-01-24/', ] - RELOAD_PROJECT_VARIANT_LOGS = [ - ('Reloading saved variants in 2 projects', None), - ('Reloading genotypes for 0 SNV_INDEL variants in family F000012_12', None), - ('Updated 0 variants in 2 families for project Test Reprocessed Project', None), - ('Reloading genotypes for 1 SNV_INDEL variants in family F000014_14', None), - ('update 1 SavedVariants', {'dbUpdate': mock.ANY}), - ('Updated 1 variants in 1 families for project Non-Analyst Project', None), - ('Reload Summary: ', None), - (' Non-Analyst Project: Updated 1 variants', None), - ] - RELOAD_SV_VARIANT_LOGS = [ - ('Updated 0 variants in 1 families for project 1kg project nรฅme with uniรงรธde', None), - ('Updated 0 variants in 1 families for project Test Reprocessed Project', None), - ('Reload Summary: ', None), - ] ADDITIONAL_SLACK_CALLS = [ mock.call( 'anvil-data-loading', @@ -977,6 +843,7 @@ def setUp(self): self.set_up() super().setUp() + @responses.activate def test_command(self, *args, **kwargs): responses.add( responses.GET, @@ -1009,10 +876,6 @@ def test_command(self, *args, **kwargs): }))) super().test_command(*args, **kwargs) - def _test_call(self, *args, reload_annotations_logs=None, reload_calls=None, **kwargs): - # No reloading for clickhouse - super()._test_call(*args, has_reload_calls=bool(reload_calls), **kwargs) - def _set_empty_loading_files(self): self.mock_subprocess.return_value.communicate.return_value = b'', b'One or more URLs matched no objects' @@ -1026,6 +889,7 @@ def _set_reloading_loading_files(self): self.mock_subprocess.side_effect = [self.mock_ls_process] def _set_loading_files(self): + responses.calls.reset() self.mock_subprocess.reset_mock() self.mock_subprocess.side_effect = [self.mock_ls_process] + [ mock_opened_file(i) for i in range(len(OPENED_RUN_JSON_FILES) - 1) @@ -1053,7 +917,7 @@ def _assert_expected_loading_file_calls(self, single_call): def _additional_loading_logs(self, data_type, version): return [(f'==> gsutil cat gs://seqr-hail-search-data/v3.1/{data_type.replace("SV", "GCNV")}/runs/{version}/metadata.json', None)] - def _assert_expected_airtable_calls(self, has_reload_calls, single_call): + def _assert_expected_airtable_calls(self, has_success_run, single_call): # Test request tracking updates for validation errors if single_call: fields = {'Status': 'Available in Seqr'} @@ -1065,8 +929,9 @@ def _assert_expected_airtable_calls(self, has_reload_calls, single_call): self.assertDictEqual(json.loads(update_loading_tracking_request.body), {'records': [ {'id': 'rec12345', 'fields': fields}, ]}) - if not has_reload_calls: - return 0, 2 + + if not has_success_run: + return 8 if single_call else 2 # Test airtable PDO updates update_pdos_request = responses.calls[1].request @@ -1099,14 +964,4 @@ def _assert_expected_airtable_calls(self, has_reload_calls, single_call): {'id': 'recfMYDEZpPtzAIeV', 'fields': {'PDOID': ['rec0ABC123']}}, ]}) - if single_call: - return 8, 0 - return 7, 2 - - def _assert_saved_variant_models_updated(self): - saved_variant = SavedVariant.objects.get(key=100, family_id=14) - self.assertDictEqual(saved_variant.genotypes, {'I000018_na21234': { - 'ab': 0.0, 'dp': 49, 'gq': 99, 'numAlt': 2, 'filters': [], - 'sampleId': 'NA21234', 'familyGuid': 'F000014_14', 'sampleType': 'WGS', 'individualGuid': 'I000018_na21234', - }}) - self.assertDictEqual(saved_variant.saved_variant_json, {}) + return 8 if single_call else 9 diff --git a/seqr/management/tests/deactivate_project_search_tests.py b/seqr/management/tests/deactivate_project_search_tests.py index cbd13ae46b..2a35dc9a4c 100644 --- a/seqr/management/tests/deactivate_project_search_tests.py +++ b/seqr/management/tests/deactivate_project_search_tests.py @@ -98,10 +98,3 @@ def _assert_expected_delete(self): 7: (0, 0, 0, 0), 8: (0, 0, 0, 0), }) - - -class HailBackendDeactivateProjectSearchTest(AuthenticationTestCase, DeactivateProjectSearchTest): - fixtures = ['users', '1kg_project'] - - CLICKHOUSE_HOSTNAME = '' - ES_HOSTNAME = '' diff --git a/seqr/management/tests/reload_saved_variant_annotations_tests.py b/seqr/management/tests/reload_saved_variant_annotations_tests.py deleted file mode 100644 index 9df44a79d6..0000000000 --- a/seqr/management/tests/reload_saved_variant_annotations_tests.py +++ /dev/null @@ -1,131 +0,0 @@ -from django.core.management import call_command -from django.core.management.base import CommandError -import json -import mock -import responses - -from seqr.views.utils.test_utils import AnvilAuthenticationTestCase -from seqr.models import Sample, SavedVariant - -MOCK_HAIL_HOST = 'test-hail-host' -MOCK_HAIL_ORIGIN = f'http://{MOCK_HAIL_HOST}' - - -@mock.patch('seqr.utils.search.hail_search_utils.HAIL_BACKEND_SERVICE_HOSTNAME', MOCK_HAIL_HOST) -class ReloadVariantAnnotationsTest(AnvilAuthenticationTestCase): - fixtures = ['users', '1kg_project'] - - CLICKHOUSE_HOSTNAME = None - - @responses.activate - def test_command(self): - responses.add(responses.POST, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup', status=200, json={ - 'results': [ - {'variantId': '1-46859832-G-A', 'updated_new_field': 'updated_value', 'rsid': 'rs123'}, - {'variantId': '1-248367227-TC-T', 'updated_field': 'updated_value'}, - ], - }) - - # Test errors - with self.assertRaises(CommandError) as ce: - call_command('reload_saved_variant_annotations') - self.assertEqual(str(ce.exception), 'Error: the following arguments are required: data_type, genome_version, chromosomes') - - with self.assertRaises(CommandError) as ce: - call_command('reload_saved_variant_annotations', 'SV', 'GRCh37') - self.assertEqual(str(ce.exception), "Error: argument data_type: invalid choice: 'SV' (choose from 'MITO', 'SNV_INDEL', 'SV_WES', 'SV_WGS')") - - # Test success - self.reset_logs() - call_command('reload_saved_variant_annotations', 'SNV_INDEL', 'GRCh37') - - self.assert_json_logs(user=None, expected=[ - ('Reloading shared annotations for 2 SNV_INDEL GRCh37 saved variants in chromosome 1 (2 unique)', None), - ('Fetched 2 additional variants in chromosome 1', None), - ('update 2 SavedVariants', {'dbUpdate': { - 'dbEntity': 'SavedVariant', - 'entityIds': ['SV0000002_1248367227_r0390_100', 'SV0059956_11560662_f019313_1'], - 'updateFields': ['saved_variant_json'], - 'updateType': 'bulk_update'}, - }), - ] + [(f'No additional SNV_INDEL GRCh37 saved variants to update in chromosome {chrom}', None) for chrom in ['2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20']] + [ - ('Reloading shared annotations for 1 SNV_INDEL GRCh37 saved variants in chromosome 21 (1 unique)', None), - ('Fetched 2 additional variants in chromosome 21', None), - ] + [(f'No additional SNV_INDEL GRCh37 saved variants to update in chromosome {chrom}', None) for chrom in ['22', 'X', 'Y', 'M']]) - - self.assertEqual(len(responses.calls), 2) - for call in responses.calls: - self.assertEqual(call.request.url, f'{MOCK_HAIL_ORIGIN}:5000/multi_lookup') - self.assertEqual(call.request.headers.get('From'), 'manage_command') - self.assertDictEqual(json.loads(responses.calls[0].request.body), { - 'genome_version': 'GRCh37', - 'data_type': 'SNV_INDEL', - 'variant_ids': [['1', 46859832, 'G', 'A'], ['1', 248367227, 'TC', 'T']], - }) - self.assertDictEqual(json.loads(responses.calls[1].request.body), { - 'genome_version': 'GRCh37', - 'data_type': 'SNV_INDEL', - 'variant_ids': [['21', 3343353, 'GAGA', 'G']], - }) - - annotation_updated_json_1 = SavedVariant.objects.get(guid='SV0000002_1248367227_r0390_100').saved_variant_json - self.assertEqual(len(annotation_updated_json_1), 17) - self.assertListEqual(annotation_updated_json_1['familyGuids'], ['F000001_1']) - self.assertEqual(annotation_updated_json_1['updated_field'], 'updated_value') - - annotation_updated_json_2 = SavedVariant.objects.get(guid='SV0059956_11560662_f019313_1').saved_variant_json - self.assertEqual(len(annotation_updated_json_2), 16) - self.assertEqual(annotation_updated_json_2['updated_new_field'], 'updated_value') - self.assertEqual(annotation_updated_json_2['rsid'], 'rs123') - self.assertEqual(annotation_updated_json_2['mainTranscriptId'], 'ENST00000505820') - self.assertEqual(len(annotation_updated_json_2['genotypes']), 3) - - # Test chromosome subset - responses.calls.reset() - self.reset_logs() - call_command('reload_saved_variant_annotations', 'SNV_INDEL', 'GRCh37', '3', '21') - - self.assert_json_logs(user=None, expected=[(log, None) for log in [ - 'No additional SNV_INDEL GRCh37 saved variants to update in chromosome 3', - 'Reloading shared annotations for 1 SNV_INDEL GRCh37 saved variants in chromosome 21 (1 unique)', - 'Fetched 2 additional variants in chromosome 21', - ]]) - - self.assertEqual(len(responses.calls), 1) - self.assertDictEqual(json.loads(responses.calls[0].request.body), { - 'genome_version': 'GRCh37', - 'data_type': 'SNV_INDEL', - 'variant_ids': [['21', 3343353, 'GAGA', 'G']], - }) - - responses.calls.reset() - self.reset_logs() - call_command('reload_saved_variant_annotations', 'SNV_INDEL', 'GRCh37', '3', '6') - self.assert_json_logs(user=None, expected=[ - (f'No additional SNV_INDEL GRCh37 saved variants to update in chromosome {chrom}', None) - for chrom in [3, 6] - ]) - self.assertEqual(len(responses.calls), 0) - - # Test SVs - responses.calls.reset() - Sample.objects.filter(guid='S000147_na21234').update(individual_id=20) - call_command('reload_saved_variant_annotations', 'SV_WGS', 'GRCh37') - - self.assertEqual(len(responses.calls), 1) - self.assertDictEqual(json.loads(responses.calls[0].request.body), { - 'genome_version': 'GRCh37', - 'data_type': 'SV_WGS', - 'variant_ids': ['prefix_19107_DEL'], - }) - - -class ClickhouseReloadVariantAnnotationsTest(AnvilAuthenticationTestCase): - - fixtures = ['users', '1kg_project'] - - def test_command(self): - with self.assertRaises(CommandError) as ce: - call_command('reload_saved_variant_annotations', 'SNV_INDEL', 'GRCh37') - self.assertEqual(str(ce.exception), 'Reloading variant annotations is not supported in clickhouse') - diff --git a/seqr/management/tests/reload_saved_variant_json_tests.py b/seqr/management/tests/reload_saved_variant_json_tests.py index 46d47bde1a..ae5983fa0f 100644 --- a/seqr/management/tests/reload_saved_variant_json_tests.py +++ b/seqr/management/tests/reload_saved_variant_json_tests.py @@ -3,22 +3,21 @@ from django.core.management import call_command, CommandError from django.test import TestCase -from seqr.models import Family PROJECT_NAME = '1kg project n\u00e5me with uni\u00e7\u00f8de' PROJECT_GUID = 'R0001_1kg' FAMILY_GUID = 'F000001_1' -@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') +@mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') class ReloadSavedVariantJsonTest(TestCase): fixtures = ['users', '1kg_project'] @mock.patch('seqr.views.utils.variant_utils.logger') - @mock.patch('seqr.views.utils.variant_utils.get_variants_for_variant_ids') + @mock.patch('seqr.utils.search.elasticsearch.es_utils.get_es_variants_for_variant_ids') def test_with_param_command(self, mock_get_variants, mock_logger): - mock_get_variants.side_effect = lambda families, variant_ids, **kwargs: \ - [{'variantId': variant_id, 'familyGuids': [family.guid for family in families]} + mock_get_variants.side_effect = lambda samples, genome_version, variant_ids, user: \ + [{'variantId': variant_id, 'familyGuids': sorted({sample.individual.family.guid for sample in samples}) if samples else ['F000012_12']} for variant_id in variant_ids] # Test with a specific project and a family id. @@ -26,9 +25,8 @@ def test_with_param_command(self, mock_get_variants, mock_logger): PROJECT_NAME, '--family-guid={}'.format(FAMILY_GUID)) - family_1 = Family.objects.get(id=1) - mock_get_variants.assert_called_with( - [family_1], ['1-46859832-G-A','21-3343353-GAGA-G'], user=None, user_email='manage_command') + mock_get_variants.assert_called_with(mock.ANY, '37', ['1-46859832-G-A','21-3343353-GAGA-G'], None) + self.assertSetEqual(set(mock_get_variants.call_args_list[0].args[0].values_list('id', flat=True)), {129, 130}) logger_info_calls = [ mock.call('Updated 2 variants in 1 families for project 1kg project n\xe5me with uni\xe7\xf8de'), @@ -43,14 +41,18 @@ def test_with_param_command(self, mock_get_variants, mock_logger): call_command('reload_saved_variant_json') self.assertEqual(mock_get_variants.call_count, 3) - family_2 = Family.objects.get(id=2) mock_get_variants.assert_has_calls([ mock.call( - [family_1, family_2], ['1-248367227-TC-T', '1-46859832-G-A', '21-3343353-GAGA-G'], user=None, user_email='manage_command', + mock.ANY, '37', ['1-248367227-TC-T', '1-46859832-G-A', '21-3343353-GAGA-G'], None, ), - mock.call([Family.objects.get(id=12)], ['1-248367227-TC-T', 'prefix_19107_DEL'], user=None, user_email='manage_command'), - mock.call([Family.objects.get(id=14)], ['1-248367227-TC-T'], user=None, user_email='manage_command') - ], any_order=True) + mock.call(mock.ANY, '37', ['1-248367227-TC-T', 'prefix_19107_DEL'], None), + mock.call(mock.ANY, '38', ['1-248367227-TC-T'], None) + ]) + self.assertSetEqual(set(mock_get_variants.call_args_list[0].args[0].values_list('id', flat=True)), { + 129, 130, 132, 133, 134, 145, 146, 148, 149, + }) + self.assertSetEqual(set(mock_get_variants.call_args_list[1].args[0].values_list('id', flat=True)), set()) + self.assertSetEqual(set(mock_get_variants.call_args_list[2].args[0].values_list('id', flat=True)), {147, 173, 174}) logger_info_calls = [ mock.call('Reloading saved variants in 4 projects'), @@ -73,7 +75,8 @@ def test_with_param_command(self, mock_get_variants, mock_logger): PROJECT_GUID, '--family-guid={}'.format(FAMILY_GUID)) - mock_get_variants.assert_called_with([family_1], ['1-46859832-G-A', '21-3343353-GAGA-G'], user=None, user_email='manage_command') + mock_get_variants.assert_called_with(mock.ANY, '37', ['1-46859832-G-A', '21-3343353-GAGA-G'], None) + self.assertSetEqual(set(mock_get_variants.call_args_list[0].args[0].values_list('id', flat=True)), {129, 130}) logger_info_calls = [ mock.call('Reload Summary: '), @@ -91,8 +94,8 @@ class ClickhouseReloadSavedVariantJsonTest(TestCase): fixtures = ['users', '1kg_project'] def test_command(self): - with self.assertRaises(CommandError) as ce: + with self.assertRaises(ValueError) as ce: call_command('reload_saved_variant_json', PROJECT_NAME) - self.assertEqual(str(ce.exception), 'Reloading variants is not supported in clickhouse') + self.assertEqual(str(ce.exception), 'handle is disabled without the elasticsearch backend') diff --git a/seqr/management/tests/transfer_families_to_different_project_tests.py b/seqr/management/tests/transfer_families_to_different_project_tests.py index 5e18dbf721..f696bbc7be 100644 --- a/seqr/management/tests/transfer_families_to_different_project_tests.py +++ b/seqr/management/tests/transfer_families_to_different_project_tests.py @@ -64,14 +64,3 @@ class TransferFamiliesClickhouseTest(TransferFamiliesTest, AnvilAuthenticationTe ('Clickhouse does not support deleting individual families from project. Manually delete SNV_INDEL data for F000002_2 in project R0001_1kg', None), ('Clickhouse does not support deleting individual families from project. Manually delete GCNV data for F000002_2 in project R0001_1kg', None), ] - - -class TransferFamiliesHailBackendTest(TransferFamiliesTest, AuthenticationTestCase): - fixtures = ['users', '1kg_project'] - - ES_HOSTNAME = '' - CLICKHOUSE_HOSTNAME = '' - - LOGS = [ - ('Disabled search for 7 samples in the following 1 families: 2', None), - ] diff --git a/seqr/utils/search/add_data_utils.py b/seqr/utils/search/add_data_utils.py index 8bd9157432..8d82255083 100644 --- a/seqr/utils/search/add_data_utils.py +++ b/seqr/utils/search/add_data_utils.py @@ -8,10 +8,7 @@ from seqr.utils.communication_utils import send_project_notification from seqr.utils.file_utils import does_file_exist from seqr.utils.logging_utils import SeqrLogger -from seqr.utils.search.utils import backend_specific_call -from seqr.utils.search.elasticsearch.es_utils import validate_es_index_metadata_and_get_samples from seqr.views.utils.airtable_utils import AirtableSession, ANVIL_REQUEST_TRACKING_TABLE -from seqr.views.utils.dataset_utils import match_and_update_search_samples, load_mapping_file from seqr.views.utils.export_utils import write_multiple_files from seqr.views.utils.pedigree_info_utils import JsonConstants from settings import SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, BASE_URL, ANVIL_UI_URL, \ @@ -20,46 +17,7 @@ logger = SeqrLogger(__name__) -def _no_es_backend_error(*args, **kwargs): - raise ValueError('Adding samples is disabled without the elasticsearch backend') - - -def add_new_es_search_samples(request_json, project, user, notify=False, expected_families=None): - dataset_type = request_json.get('datasetType') - if dataset_type not in Sample.DATASET_TYPE_LOOKUP: - raise ValueError(f'Invalid dataset type "{dataset_type}"') - - sample_ids, sample_type, sample_data = backend_specific_call( - validate_es_index_metadata_and_get_samples, - _no_es_backend_error, - _no_es_backend_error, - )(request_json, project) - if not sample_ids: - raise ValueError('No samples found. Make sure the specified caller type is correct') - - sample_id_to_individual_id_mapping = load_mapping_file( - request_json['mappingFilePath'], user) if request_json.get('mappingFilePath') else {} - ignore_extra_samples = request_json.get('ignoreExtraSamplesInCallset') - sample_project_tuples = [(sample_id, project.name) for sample_id in sample_ids] - new_samples, updated_samples, inactivated_sample_guids, updated_family_guids = match_and_update_search_samples( - projects=[project], - user=user, - sample_project_tuples=sample_project_tuples, - sample_data=sample_data, - sample_type=sample_type, - dataset_type=dataset_type, - expected_families=expected_families, - sample_id_to_individual_id_mapping=sample_id_to_individual_id_mapping, - raise_unmatched_error_template=None if ignore_extra_samples else 'Matches not found for sample ids: {sample_ids}. Uploading a mapping file for these samples, or select the "Ignore extra samples in callset" checkbox to ignore.' - ) - - if notify: - _basic_notify_search_data_loaded(project, dataset_type, sample_type, new_samples.values_list('sample_id')) - - return inactivated_sample_guids, updated_family_guids, updated_samples - - -def _basic_notify_search_data_loaded(project, dataset_type, sample_type, new_samples, email_template=None, slack_channel=None, include_slack_detail=False): +def basic_notify_search_data_loaded(project, dataset_type, sample_type, new_samples, email_template=None, slack_channel=None, include_slack_detail=False): msg_dataset_type = '' if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS else f' {dataset_type}' num_new_samples = len(new_samples) sample_summary = f'{num_new_samples} new {sample_type}{msg_dataset_type} samples' @@ -87,7 +45,7 @@ def notify_search_data_loaded(project, is_internal, dataset_type, sample_type, n 'Let us know if you have any questions.', ]) - _basic_notify_search_data_loaded( + basic_notify_search_data_loaded( project, dataset_type, sample_type, new_samples, email_template=email_template, slack_channel=SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL if is_internal else SEQR_SLACK_ANVIL_DATA_LOADING_CHANNEL, include_slack_detail=is_internal, diff --git a/seqr/utils/search/elasticsearch/es_utils.py b/seqr/utils/search/elasticsearch/es_utils.py index 2381a62b42..5fe4a68106 100644 --- a/seqr/utils/search/elasticsearch/es_utils.py +++ b/seqr/utils/search/elasticsearch/es_utils.py @@ -4,7 +4,7 @@ import elasticsearch_dsl from urllib3.connectionpool import connection_from_url -from seqr.models import Sample +from seqr.models import Sample, SavedVariant from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_set_json from seqr.utils.search.constants import VCF_FILE_EXTENSIONS, XPOS_SORT_KEY from seqr.utils.search.elasticsearch.es_gene_agg_search import EsGeneAggSearch @@ -29,6 +29,8 @@ class InvalidIndexException(Exception): } ES_ERROR_LOG_EXCEPTIONS = {InvalidIndexException} +MAX_VARIANTS_FETCH = 1000 + def _get_transport_error_type(error): error_type = 'no detail' @@ -206,7 +208,7 @@ def get_elasticsearch_status(): errors = ['{} does not exist and is used by project(s) {}'.format( index, ', '.join(['{} ({} samples)'.format(p.name, len(indivs)) for p, indivs in project_individuals.items()]) - ) for index, project_individuals in seqr_index_projects.items() if project_individuals] + ) for index, project_individuals in sorted(seqr_index_projects.items()) if project_individuals] return { 'indices': indices, @@ -263,11 +265,12 @@ def _get_es_indices(client): return indices, seqr_index_projects -def get_es_variants_for_variant_ids(samples, genome_version, variants_by_id, user, return_all_queried_families=False, **kwargs): +def get_es_variants_for_variant_ids(samples, genome_version, variant_ids, user): + variant_ids = sorted(set(variant_ids)) variants = EsSearch( - samples, genome_version, user=user, return_all_queried_families=return_all_queried_families, sort=XPOS_SORT_KEY, - ).filter_by_variant_ids(list(variants_by_id.keys())) - return variants.search(num_results=len(variants_by_id)) + samples, genome_version, user=user, sort=XPOS_SORT_KEY, + ).filter_by_variant_ids(variant_ids) + return variants.search(num_results=len(variant_ids)) def get_es_variants(samples, search, user, previous_search_results, genome_version, sort=None, page=None, num_results=None, @@ -290,8 +293,8 @@ def get_es_variants(samples, search, user, previous_search_results, genome_versi in_silico=search.get('in_silico'), quality_filter=search.get('qualityFilter'), custom_query=search.get('customQuery'), locus=search.get('locus'), skip_genotype_filter=skip_genotype_filter, dataset_type=search.get('dataset_type'), secondary_dataset_type=search.get('secondary_dataset_type'), - **search.get('parsed_locus') - + genes=search.get('genes'), intervals=search.get('intervals'), rs_ids=search.get('rs_ids'), + variant_ids=search.get('variant_ids'), exclude_locations=search.get('exclude_locations'), ) return es_search.search(page=page, num_results=num_results) @@ -330,3 +333,36 @@ def process_es_previously_loaded_gene_aggs(previous_search_results): for family_guid in variants[0]['familyGuids']: gene_aggs[gene_id]['families'][family_guid] += len(variants) return gene_aggs + + +def update_project_saved_variant_json(project_id, genome_version, family_guids=None, user=None, **kwargs): + from seqr.views.utils.variant_utils import get_saved_variants + saved_variants = get_saved_variants(genome_version, project_id, family_guids).select_related('family') + + if not saved_variants: + return None + + family_ids = set() + variant_ids = set() + saved_variants_map = {} + for v in saved_variants: + family_ids.add(v.family_id) + variant_ids.add(v.variant_id) + saved_variants_map[(v.variant_id, v.family.guid)] = v + + variant_ids = sorted(variant_ids) + samples = Sample.objects.filter(individual__family_id__in=family_ids, is_active=True) + variants_json = [] + for sub_var_ids in [variant_ids[i:i+MAX_VARIANTS_FETCH] for i in range(0, len(variant_ids), MAX_VARIANTS_FETCH)]: + variants_json += get_es_variants_for_variant_ids(samples, genome_version, sub_var_ids, user) + + updated_saved_variants = {} + for var in variants_json: + for family_guid in var['familyGuids']: + saved_variant = saved_variants_map.get((var['variantId'], family_guid)) + if saved_variant: + saved_variant.saved_variant_json = var + updated_saved_variants[saved_variant.guid] = saved_variant + SavedVariant.bulk_update_models(user, list(updated_saved_variants.values()), ['saved_variant_json']) + + return updated_saved_variants diff --git a/seqr/utils/search/elasticsearch/es_utils_tests.py b/seqr/utils/search/elasticsearch/es_utils_tests.py index 787baed991..e6be5cc0a2 100644 --- a/seqr/utils/search/elasticsearch/es_utils_tests.py +++ b/seqr/utils/search/elasticsearch/es_utils_tests.py @@ -13,7 +13,7 @@ from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.utils.search.utils import get_single_variant, query_variants, \ - get_variant_query_gene_counts, get_variants_for_variant_ids, InvalidSearchException + get_variant_query_gene_counts, get_es_variants_for_variant_ids, InvalidSearchException from seqr.utils.search.elasticsearch.es_search import _get_family_affected_status from seqr.utils.search.elasticsearch.es_utils import InvalidIndexException from seqr.utils.search.utils import _get_liftover @@ -1394,9 +1394,10 @@ def assertCachedResults(self, results_model, expected_results, sort='xpos'): @urllib3_responses.activate def test_get_es_variants_for_variant_ids(self): setup_responses() - get_variants_for_variant_ids(self.families, ['2-103343353-GAGA-G', '1-248367227-TC-T', 'prefix-938_DEL']) + samples = Sample.objects.filter(individual__family__in=self.families).exclude(dataset_type='MITO') + get_es_variants_for_variant_ids(samples, '37', ['2-103343353-GAGA-G', '1-248367227-TC-T', 'prefix-938_DEL'], None) self.assertExecutedSearch( - filters=[{'terms': {'variantId': ['2-103343353-GAGA-G', '1-248367227-TC-T', 'prefix-938_DEL']}}], + filters=[{'terms': {'variantId': ['1-248367227-TC-T', '2-103343353-GAGA-G', 'prefix-938_DEL']}}], size=6, index=','.join([INDEX_NAME, SV_INDEX_NAME]), ) @@ -1513,6 +1514,7 @@ def test_invalid_get_es_variants(self, mock_logger): query_variants(results_model) self.assertEqual(str(cm.exception), 'Could not find expected indices: test_index_sv, test_index_mito_wgs, test_index') + @mock.patch('seqr.utils.search.utils.MAX_VARIANTS', 100) @urllib3_responses.activate def test_get_es_variants(self): setup_responses() @@ -1554,16 +1556,14 @@ def test_get_es_variants(self): # test load_all setup_responses() - with mock.patch('seqr.utils.search.utils.MAX_VARIANTS', 100): - variants, _ = query_variants(results_model, page=1, num_results=2, load_all=True) + variants, _ = query_variants(results_model, page=1, num_results=2, load_all=True) self.assertExecutedSearch(filters=[ANNOTATION_QUERY, ALL_INHERITANCE_QUERY], start_index=4, size=1) self.assertEqual(len(variants), 5) self.assertListEqual(variants, PARSED_VARIANTS + PARSED_VARIANTS + PARSED_VARIANTS[:1]) # test does not re-fetch once all loaded urllib3_responses.reset() - with mock.patch('seqr.utils.search.utils.MAX_VARIANTS', 1): - variants, _ = query_variants(results_model, page=1, num_results=2, load_all=True) + variants, _ = query_variants(results_model, page=1, num_results=2, load_all=True) self.assertEqual(len(variants), 5) self.assertListEqual(variants, PARSED_VARIANTS + PARSED_VARIANTS + PARSED_VARIANTS[:1]) @@ -2859,11 +2859,11 @@ def test_multi_project_prefilter_indices_get_es_variants(self): _, total_results = query_variants(results_model, num_results=2) self.assertEqual(total_results, 14) - self.assertTrue('index_metadata__{},{},{}'.format(INDEX_NAME, MITO_WGS_INDEX_NAME, SV_INDEX_NAME) in REDIS_CACHE) + self.assertTrue('index_metadata__{},{},{}'.format(INDEX_NAME, SECOND_INDEX_NAME, SV_INDEX_NAME) in REDIS_CACHE) gene_filter = {'terms': {'geneIds': ['ENSG00000228198']}} prefilter_search = dict( - filters=[gene_filter], index=f'{SV_INDEX_NAME},{MITO_WGS_INDEX_NAME},{SECOND_INDEX_NAME},{INDEX_NAME}', + filters=[gene_filter], index=f'{SV_INDEX_NAME},{SECOND_INDEX_NAME},{INDEX_NAME}', size=200, expected_source_fields=set(), ) sv_search = dict( @@ -2910,24 +2910,6 @@ def test_multi_project_prefilter_indices_get_es_variants(self): '_name': 'F000011_11' }} ], start_index=0, size=2, index=SECOND_INDEX_NAME), - dict(filters=[ - gene_filter, - { - 'bool': {'must': [ - {'bool': {'should': [ - {'term': {'samples_num_alt_1': 'HG00731'}}, - {'term': {'samples_num_alt_2': 'HG00731'}}, - ]}}, {'bool': {'must_not': [ - {'term': {'samples_gq_0_to_5': 'HG00731'}}, - {'term': {'samples_gq_5_to_10': 'HG00731'}}, - {'term': {'samples_gq_0_to_5': 'HG00732'}}, - {'term': {'samples_gq_5_to_10': 'HG00732'}}, - {'term': {'samples_gq_0_to_5': 'HG00733'}}, - {'term': {'samples_gq_5_to_10': 'HG00733'}}, - ]}} - ], - '_name': 'F000002_2' - }}], start_index=0, size=2, index=MITO_WGS_INDEX_NAME), dict(filters=[ gene_filter, {'bool': {'should': [ @@ -3018,13 +3000,14 @@ def test_skip_genotype_filter(self): expected_transcript_variant['selectedMainTranscriptId'] = PARSED_VARIANTS[1]['selectedMainTranscriptId'] self.assertListEqual(variants, [expected_transcript_variant, PARSED_MULTI_INDEX_VARIANT]) self.assertExecutedSearch( - index=','.join([INDEX_NAME, MITO_WGS_INDEX_NAME, SECOND_INDEX_NAME]), + index=','.join([INDEX_NAME, SECOND_INDEX_NAME]), filters=[{'terms': {'geneIds': ['ENSG00000228198']}}, ANNOTATION_QUERY], size=3, ) # test with inheritance override search_model.search['inheritance'] = {'mode': 'any_affected'} + search_model.search['locus'] = None search_model.save() _set_cache('search_results__{}__xpos'.format(results_model.guid), None) query_variants(results_model, num_results=2, skip_genotype_filter=True) @@ -3032,7 +3015,6 @@ def test_skip_genotype_filter(self): self.assertExecutedSearches([ dict( filters=[ - {'terms': {'geneIds': ['ENSG00000228198']}}, ANNOTATION_QUERY, {'bool': { 'should': [ @@ -3044,7 +3026,6 @@ def test_skip_genotype_filter(self): ], start_index=0, size=2, index=SECOND_INDEX_NAME), dict( filters=[ - {'terms': {'geneIds': ['ENSG00000228198']}}, ANNOTATION_QUERY, {'bool': { 'should': [ @@ -3056,7 +3037,6 @@ def test_skip_genotype_filter(self): ], start_index=0, size=2, index=MITO_WGS_INDEX_NAME), dict( filters=[ - {'terms': {'geneIds': ['ENSG00000228198']}}, ANNOTATION_QUERY, {'bool': { 'should': [ diff --git a/seqr/utils/search/hail_search_utils.py b/seqr/utils/search/hail_search_utils.py deleted file mode 100644 index 0cc5f53226..0000000000 --- a/seqr/utils/search/hail_search_utils.py +++ /dev/null @@ -1,175 +0,0 @@ -from collections import defaultdict - -from django.db.models import F, Min, Case, When -from time import sleep -from urllib3.connectionpool import connection_from_url - -import requests -from reference_data.models import Omim, GeneConstraint, GENOME_VERSION_LOOKUP -from seqr.models import Sample, PhenotypePrioritization, Individual -from seqr.utils.search.constants import PRIORITIZED_GENE_SORT, X_LINKED_RECESSIVE -from settings import HAIL_BACKEND_SERVICE_HOSTNAME, HAIL_BACKEND_SERVICE_PORT - - -def _hail_backend_url(path): - return f'http://{HAIL_BACKEND_SERVICE_HOSTNAME}:{HAIL_BACKEND_SERVICE_PORT}/{path}' - -def _execute_search(search_body, user, path='search', exception_map=None, user_email=None, max_retries=0): - for _ in range(max_retries + 1): - response = requests.post(_hail_backend_url(path), json=search_body, headers={'From': user_email or user.email}, timeout=300) - if response.status_code == 200: - return response.json() - sleep(1) - - error = (exception_map or {}).get(response.status_code) or response.text or response.reason - raise requests.HTTPError(error, response=response) - - -def ping_hail_backend(): - response = connection_from_url(_hail_backend_url('status')).urlopen('HEAD', '/status', timeout=5, retries=3) - if response.status >= 400: - raise requests.HTTPError(f'{response.status}: {response.reason or response.text}', response=response) - - -def get_hail_variants(samples, search, user, previous_search_results, genome_version, sort=None, page=1, num_results=100, - gene_agg=False, **kwargs): - end_offset = num_results * page - search_body = _format_search_body(samples, genome_version, end_offset, search) - - frequencies = search_body.pop('freqs', None) - if frequencies and frequencies.get('callset'): - frequencies['seqr'] = frequencies.pop('callset') - - search_body.update({ - 'sort': sort, - 'sort_metadata': _get_sort_metadata(sort, samples), - 'frequencies': frequencies, - 'quality_filter': search_body.pop('qualityFilter', None), - **search_body.pop('parsed_locus') - }) - search_body.pop('skipped_samples', None) - - sv_data_types = [data_type for data_type in search_body['sample_data'] if data_type.startswith(Sample.DATASET_TYPE_SV_CALLS)] - if len(sv_data_types) > 1 and not gene_agg: - response_json = _execute_multi_sample_types_searches(sv_data_types, search_body, user) - else: - path = 'gene_counts' if gene_agg else 'search' - response_json = _execute_search(search_body, user, path) - - if gene_agg: - previous_search_results['gene_aggs'] = response_json - return response_json - - previous_search_results['total_results'] = response_json['total'] - previous_search_results['all_results'] = response_json['results'] - return response_json['results'][end_offset - num_results:end_offset] - - -def _execute_multi_sample_types_searches(sv_data_types, search_body, user): - sample_data_by_sample_data_type = defaultdict(lambda: defaultdict(list)) - for data_type, samples in search_body['sample_data'].items(): - if data_type in sv_data_types: - sample_type = samples[0]['sample_type'] - sample_data_by_sample_data_type[sample_type][data_type] = samples - else: - for sample in samples: - sample_data_by_sample_data_type[sample['sample_type']][data_type].append(sample) - - total = 0 - results = [] - for sample_type in sorted(sample_data_by_sample_data_type.keys()): - search_body['sample_data'] = sample_data_by_sample_data_type[sample_type] - sample_response_json = _execute_search(search_body, user) - total += sample_response_json['total'] - results += sample_response_json['results'] - - return {'total': total, 'results': sorted(results, key=lambda x: x[0]['_sort'] if isinstance(x, list) else x['_sort'])[:search_body['num_results']]} - - -def get_hail_variants_for_variant_ids(samples, genome_version, parsed_variant_ids, user, user_email=None): - search = { - 'variant_ids': [parsed_id for parsed_id in parsed_variant_ids.values() if parsed_id], - 'variant_keys': [variant_id for variant_id, parsed_id in parsed_variant_ids.items() if not parsed_id], - } - search_body = _format_search_body(samples, genome_version, len(parsed_variant_ids), search) - response_json = _execute_search(search_body, user, user_email=user_email) - - return response_json['results'] - - -def hail_variant_lookup(user, variant_id, data_type, genome_version=None, samples=None, **kwargs): - body = { - 'variant_id': variant_id, - 'data_type': data_type, - 'genome_version': GENOME_VERSION_LOOKUP[genome_version], - **kwargs, - } - if samples: - body['sample_data'] = _get_sample_data(samples).pop(data_type) - return _execute_search(body, user, path='lookup', exception_map={404: 'Variant not present in seqr'}) - - -def hail_variant_multi_lookup(user_email, variant_ids, data_type, genome_version): - body = {'genome_version': genome_version, 'data_type': data_type, 'variant_ids': variant_ids} - response_json = _execute_search(body, user=None, user_email=user_email, path='multi_lookup', max_retries=2) - return response_json['results'] - - -def _format_search_body(samples, genome_version, num_results, search): - search_body = { - 'genome_version': GENOME_VERSION_LOOKUP[genome_version], - 'num_results': num_results, - } - search_body.update(search) - search_body['sample_data'] = _get_sample_data(samples, **search_body) - search_body.pop('dataset_type', None) - search_body.pop('secondary_dataset_type', None) - return search_body - - -def search_data_type(dataset_type, sample_type): - return f'{dataset_type}_{sample_type}' if dataset_type == Sample.DATASET_TYPE_SV_CALLS else dataset_type - - -def _get_sample_data(samples, inheritance_filter=None, inheritance_mode=None, **kwargs): - sample_values = dict( - individual_guid=F('individual__guid'), - family_guid=F('individual__family__guid'), - project_guid=F('individual__family__project__guid'), - affected=F('individual__affected'), - ) - if inheritance_mode == X_LINKED_RECESSIVE: - sample_values['is_male'] = Case(When(individual__sex__in=Individual.MALE_SEXES, then=True), default=False) - sample_data = samples.order_by('guid').values('individual__individual_id', 'dataset_type', 'sample_type', **sample_values) - - custom_affected = (inheritance_filter or {}).pop('affected', None) - if custom_affected: - for s in sample_data: - s['affected'] = custom_affected.get(s['individual_guid']) or s['affected'] - - sample_data_by_data_type = defaultdict(list) - for s in sample_data: - dataset_type = s.pop('dataset_type') - s['sample_id'] = s.pop('individual__individual_id') # Note: set sample_id to individual_id - data_type_key = search_data_type(dataset_type, s['sample_type']) - sample_data_by_data_type[data_type_key].append(s) - - return sample_data_by_data_type - - -def _get_sort_metadata(sort, samples): - sort_metadata = None - if sort == 'in_omim': - sort_metadata = list(Omim.objects.filter(phenotype_mim_number__isnull=False, gene__isnull=False).values_list('gene__gene_id', flat=True)) - elif sort == 'constraint': - sort_metadata = { - agg['gene__gene_id']: agg['mis_z_rank'] + agg['pLI_rank'] for agg in - GeneConstraint.objects.values('gene__gene_id', 'mis_z_rank', 'pLI_rank') - } - elif sort == PRIORITIZED_GENE_SORT: - sort_metadata = { - agg['gene_id']: agg['min_rank'] for agg in PhenotypePrioritization.objects.filter( - individual__family_id=samples[0].individual.family_id, rank__lte=100, - ).values('gene_id').annotate(min_rank=Min('rank')) - } - return sort_metadata diff --git a/seqr/utils/search/hail_search_utils_tests.py b/seqr/utils/search/hail_search_utils_tests.py deleted file mode 100644 index b1e1700687..0000000000 --- a/seqr/utils/search/hail_search_utils_tests.py +++ /dev/null @@ -1,381 +0,0 @@ -from copy import deepcopy -from django.test import TestCase -import json -import mock -from requests import HTTPError -import responses - -from seqr.models import Family, Project -from seqr.utils.search.utils import get_variant_query_gene_counts, query_variants, get_single_variant, \ - get_variants_for_variant_ids, variant_lookup, sv_variant_lookup, InvalidSearchException -from seqr.utils.search.search_utils_tests import SearchTestHelper -from hail_search.test_utils import get_hail_search_body, EXPECTED_SAMPLE_DATA, FAMILY_1_SAMPLE_DATA, \ - ALL_AFFECTED_SAMPLE_DATA, CUSTOM_AFFECTED_SAMPLE_DATA, HAIL_BACKEND_VARIANTS, \ - LOCATION_SEARCH, EXCLUDE_LOCATION_SEARCH, VARIANT_ID_SEARCH, RSID_SEARCH, GENE_COUNTS, FAMILY_2_VARIANT_SAMPLE_DATA, \ - FAMILY_2_MITO_SAMPLE_DATA, EXPECTED_SAMPLE_DATA_WITH_SEX, VARIANT_LOOKUP_VARIANT, FAMILY_11_SAMPLE_WGS, \ - GCNV_VARIANT4, SV_VARIANT2 - -MOCK_HOST = 'test-hail-host' -MOCK_ORIGIN = f'http://{MOCK_HOST}' - -SV_WGS_SAMPLE_DATA = [{ - 'individual_guid': 'I000018_na21234', 'family_guid': 'F000014_14', 'project_guid': 'R0004_non_analyst_project', - 'affected': 'A', 'sample_id': 'NA21234', 'sample_type': 'WGS', -}, { - 'individual_guid': 'I000019_na21987', 'family_guid': 'F000014_14', 'project_guid': 'R0004_non_analyst_project', - 'affected': 'A', 'sample_id': 'NA21987', 'sample_type': 'WGS', -}, { - 'individual_guid': 'I000021_na21654', 'family_guid': 'F000014_14', 'project_guid': 'R0004_non_analyst_project', - 'affected': 'N', 'sample_id': 'NA21654', 'sample_type': 'WGS', -}] -SV_WES_SAMPLE_DATA = FAMILY_2_VARIANT_SAMPLE_DATA['SNV_INDEL'] - -EXPECTED_MITO_SAMPLE_DATA = deepcopy(FAMILY_2_MITO_SAMPLE_DATA) -EXPECTED_MITO_SAMPLE_DATA['MITO'][0].update({'individual_guid': 'I000004_hg00731', 'sample_id': 'HG00731', 'affected': 'A'}) - -ALL_EXPECTED_SAMPLE_DATA = {**EXPECTED_SAMPLE_DATA, **EXPECTED_MITO_SAMPLE_DATA} - - -@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') -@mock.patch('seqr.utils.search.hail_search_utils.HAIL_BACKEND_SERVICE_HOSTNAME', MOCK_HOST) -class HailSearchUtilsTests(SearchTestHelper, TestCase): - databases = ['default', 'reference_data'] - fixtures = ['users', '1kg_project', 'reference_data'] - - def setUp(self): - Project.objects.update(genome_version='37') - super(HailSearchUtilsTests, self).set_up() - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/search', status=200, json={ - 'results': HAIL_BACKEND_VARIANTS, 'total': 5, - }) - - def _test_minimal_search_call(self, expected_search_body=None, call_offset=-1, url_path='search', sample_data=ALL_EXPECTED_SAMPLE_DATA, **kwargs): - expected_search = expected_search_body or get_hail_search_body(genome_version='GRCh37', sample_data=sample_data, **kwargs) - - executed_request = responses.calls[call_offset].request - self.assertEqual(executed_request.headers.get('From'), 'test_user@broadinstitute.org') - self.assertEqual(executed_request.url.split('/')[-1], url_path) - self.assertDictEqual(json.loads(executed_request.body), expected_search) - - def _test_expected_search_call(self, search_fields=None, gene_ids=None, intervals=None, exclude_intervals=False, - rs_ids=None, variant_ids=None, dataset_type=None, secondary_dataset_type=None, - frequencies=None, inheritance_mode='de_novo', inheritance_filter=None, - quality_filter=None, sort='xpos', sort_metadata=None, annotations=None, - annotations_secondary=None, **kwargs): - - expected_search = { - 'sort': sort, - 'sort_metadata': sort_metadata, - 'inheritance_mode': inheritance_mode, - 'inheritance_filter': inheritance_filter or {}, - 'frequencies': frequencies, - 'quality_filter': quality_filter, - 'intervals': intervals, - 'exclude_intervals': exclude_intervals, - 'gene_ids': gene_ids, - 'variant_ids': variant_ids, - 'rs_ids': rs_ids, - } - if annotations: - expected_search['annotations'] = annotations - if annotations_secondary: - expected_search['annotations_secondary'] = annotations_secondary - expected_search.update({field: self.search_model.search[field] for field in search_fields or []}) - - self._test_minimal_search_call(**expected_search, **kwargs) - - @mock.patch('seqr.utils.search.utils.MAX_FAMILY_COUNTS', {'WES': 2, 'WGS': 1}) - @responses.activate - def test_query_variants(self): - variants, total = query_variants(self.results_model, user=self.user) - self.assertListEqual(variants, HAIL_BACKEND_VARIANTS) - self.assertEqual(total, 5) - self.assert_cached_results({'all_results': HAIL_BACKEND_VARIANTS, 'total_results': 5}) - self._test_expected_search_call() - - variants, _ = query_variants( - self.results_model, user=self.user, sort='cadd', skip_genotype_filter=True, page=2, num_results=1, - ) - self.assertListEqual(variants, HAIL_BACKEND_VARIANTS[1:]) - self._test_expected_search_call(sort='cadd', num_results=2) - - raw_variant_locus = '1-10439-AC-A,1-91511686-TCA-G' - self.search_model.search['locus'] = {'rawVariantItems': raw_variant_locus} - query_variants(self.results_model, user=self.user, sort='in_omim') - self._test_expected_search_call( - num_results=2, dataset_type='SNV_INDEL', sample_data={'SNV_INDEL': EXPECTED_SAMPLE_DATA['SNV_INDEL']}, - sort='in_omim', sort_metadata=['ENSG00000240361', 'ENSG00000135953'], - **VARIANT_ID_SEARCH, - ) - - self.search_model.search['locus']['rawVariantItems'] = 'rs1801131' - query_variants(self.results_model, user=self.user, sort='constraint') - self._test_expected_search_call( - sort='constraint', sort_metadata={'ENSG00000223972': 2}, **RSID_SEARCH, - sample_data={'SNV_INDEL': EXPECTED_SAMPLE_DATA['SNV_INDEL'], **EXPECTED_MITO_SAMPLE_DATA}, - ) - - raw_locus = 'CDC7, chr2:1234-5678, chr7:100-10100%10, ENSG00000177000' - self.search_model.search['locus']['rawItems'] = raw_locus - query_variants(self.results_model, user=self.user) - self._test_expected_search_call(**LOCATION_SEARCH, sample_data=EXPECTED_SAMPLE_DATA) - - locus = self.search_model.search.pop('locus') - self.search_model.search['exclude'] = {'rawItems': locus['rawItems']} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call(exclude={}, **EXCLUDE_LOCATION_SEARCH) - - self.search_model.search = { - 'inheritance': {'mode': 'recessive', 'filter': {'affected': { - 'I000004_hg00731': 'N', 'I000005_hg00732': 'A', 'I000006_hg00733': 'U', - }}}, 'annotations': {'frameshift': ['frameshift_variant']}, - } - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - inheritance_mode='recessive', dataset_type='SNV_INDEL', secondary_dataset_type=None, - search_fields=['annotations'], sample_data=CUSTOM_AFFECTED_SAMPLE_DATA, - ) - - self.search_model.search['inheritance']['filter'] = {} - self.search_model.search['annotations_secondary'] = self.search_model.search['annotations'] - sv_annotations = {'structural_consequence': ['LOF']} - self.search_model.search['annotations'] = sv_annotations - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - inheritance_mode='recessive', dataset_type='SV', secondary_dataset_type='SNV_INDEL', - search_fields=['annotations', 'annotations_secondary'], sample_data=EXPECTED_SAMPLE_DATA, - ) - - self.search_model.search['annotations'] = self.search_model.search['annotations_secondary'] - self.search_model.search['annotations_secondary'] = sv_annotations - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - inheritance_mode='recessive', dataset_type='SNV_INDEL', secondary_dataset_type='SV', - search_fields=['annotations', 'annotations_secondary'] - ) - - screen_annotations = {'SCREEN': ['dELS', 'DNase-only']} - self.search_model.search['annotations_secondary'].update(screen_annotations) - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - inheritance_mode='recessive', dataset_type='SNV_INDEL', secondary_dataset_type='ALL', - search_fields=['annotations', 'annotations_secondary'] - ) - - self.search_model.search['annotations_secondary']['structural_consequence'] = [] - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - inheritance_mode='recessive', dataset_type='SNV_INDEL', secondary_dataset_type='SNV_INDEL', - search_fields=['annotations'], annotations_secondary=screen_annotations, omit_data_type='SV_WES', - ) - - self.search_model.search['inheritance']['mode'] = 'x_linked_recessive' - query_variants(self.results_model, user=self.user) - self._test_expected_search_call( - inheritance_mode='x_linked_recessive', dataset_type='SNV_INDEL', secondary_dataset_type='SNV_INDEL', - search_fields=['annotations'], annotations_secondary=screen_annotations, sample_data=EXPECTED_SAMPLE_DATA_WITH_SEX, - omit_data_type='SV_WES', - ) - - self.results_model.families.set(Family.objects.filter(id__in=[2, 11, 14])) - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Location must be specified to search across multiple projects') - - self.search_model.search = {'inheritance': {'mode': 'de_novo'}, 'annotations': {'structural_consequence': ['LOF']}} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call(search_fields=['annotations'], dataset_type='SV', call_offset=-2, sample_data={ - 'SV_WES': SV_WES_SAMPLE_DATA, - }) - self._test_expected_search_call(search_fields=['annotations'], dataset_type='SV', call_offset=-1, sample_data={ - 'SV_WGS': SV_WGS_SAMPLE_DATA, - }) - - del self.search_model.search['annotations'] - self.search_model.search['locus'] = {'rawVariantItems': raw_variant_locus} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call(**VARIANT_ID_SEARCH, num_results=2, dataset_type='SNV_INDEL', sample_data={ - 'SNV_INDEL': FAMILY_2_VARIANT_SAMPLE_DATA['SNV_INDEL'] + [FAMILY_11_SAMPLE_WGS], - }) - - self.search_model.search['locus'] = {'rawItems': 'M:10-100 '} - query_variants(self.results_model, user=self.user) - self._test_expected_search_call(intervals=[['M', 10, 100]], sample_data=EXPECTED_MITO_SAMPLE_DATA) - - self.search_model.search['locus']['rawItems'] += raw_locus - variants, total = query_variants(self.results_model, user=self.user) - self.assertListEqual(variants, [ - HAIL_BACKEND_VARIANTS[0], HAIL_BACKEND_VARIANTS[0], HAIL_BACKEND_VARIANTS[1], HAIL_BACKEND_VARIANTS[1], - ]) - self.assertEqual(total, 10) - expected_search = { - 'gene_ids': LOCATION_SEARCH['gene_ids'], - 'intervals': [['M', 10, 100]] + LOCATION_SEARCH['intervals'], - } - self._test_expected_search_call(**expected_search, call_offset=-2, sample_data={ - **FAMILY_2_VARIANT_SAMPLE_DATA, 'SV_WES': SV_WES_SAMPLE_DATA, **EXPECTED_MITO_SAMPLE_DATA, - }) - self._test_expected_search_call(**expected_search, call_offset=-1, sample_data={ - 'SNV_INDEL': [FAMILY_11_SAMPLE_WGS], 'SV_WGS': SV_WGS_SAMPLE_DATA, - }) - - self.search_model.search['locus']['rawItems'] = raw_locus - query_variants(self.results_model, user=self.user) - self._test_expected_search_call(**LOCATION_SEARCH, call_offset=-2, sample_data={ - **FAMILY_2_VARIANT_SAMPLE_DATA, 'SV_WES': SV_WES_SAMPLE_DATA, - }) - self._test_expected_search_call(**LOCATION_SEARCH, call_offset=-1, sample_data={ - 'SNV_INDEL': [FAMILY_11_SAMPLE_WGS], 'SV_WGS': SV_WGS_SAMPLE_DATA, - }) - - self.results_model.families.set(Family.objects.filter(project_id=1)) - query_variants(self.results_model, user=self.user) - self._test_expected_search_call(**LOCATION_SEARCH, sample_data={ - 'SNV_INDEL': FAMILY_1_SAMPLE_DATA['SNV_INDEL'] + EXPECTED_SAMPLE_DATA['SNV_INDEL'], - 'SV_WES': SV_WES_SAMPLE_DATA, - }) - - del self.search_model.search['locus'] - with self.assertRaises(InvalidSearchException) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(str(cm.exception), 'Location must be specified to search across multiple families in large projects') - - quality_filter = {'min_ab': 10, 'min_gq': 15, 'vcf_filter': 'pass'} - freq_filter = {'callset': {'af': 0.1}, 'gnomad_genomes': {'af': 0.01, 'ac': 3, 'hh': 3}} - genotype_filter = {'genotype': {'I000001_na19675': 'ref_alt'}} - self.search_model.search = deepcopy({ - 'inheritance': {'mode': 'any_affected', 'filter': genotype_filter}, - 'freqs': freq_filter, - 'qualityFilter': quality_filter, - 'in_silico': {'cadd': '11.5', 'sift': 'D'}, - }) - self.results_model.families.set(Family.objects.filter(guid='F000001_1')) - query_variants(self.results_model, user=self.user, sort='prioritized_gene') - expected_freq_filter = {'seqr': freq_filter['callset'], 'gnomad_genomes': freq_filter['gnomad_genomes']} - self._test_expected_search_call( - inheritance_mode=None, inheritance_filter=genotype_filter, sample_data=FAMILY_1_SAMPLE_DATA, - search_fields=['in_silico'], frequencies=expected_freq_filter, quality_filter=quality_filter, - sort='prioritized_gene', sort_metadata={'ENSG00000268903': 1, 'ENSG00000268904': 11}, - ) - - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/search', status=400, body='Bad Search Error') - with self.assertRaises(HTTPError) as cm: - query_variants(self.results_model, user=self.user) - self.assertEqual(cm.exception.response.status_code, 400) - self.assertEqual(str(cm.exception), 'Bad Search Error') - - @responses.activate - def test_get_variant_query_gene_counts(self): - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/gene_counts', json=GENE_COUNTS, status=200) - - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, GENE_COUNTS) - self.assert_cached_results({'gene_aggs': gene_counts}) - self._test_expected_search_call(url_path='gene_counts', sort=None) - - @responses.activate - def test_variant_lookup(self): - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/lookup', status=200, json=VARIANT_LOOKUP_VARIANT) - variant = variant_lookup(self.user, ('1', 10439, 'AC', 'A'), genome_version='37', foo='bar') - self.assertDictEqual(variant, VARIANT_LOOKUP_VARIANT) - self._test_minimal_search_call(url_path='lookup', expected_search_body={ - 'variant_id': ['1', 10439, 'AC', 'A'], 'genome_version': 'GRCh37', 'foo': 'bar', 'data_type': 'SNV_INDEL', - }) - - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/lookup', status=404) - with self.assertRaises(HTTPError) as cm: - variant_lookup(self.user, ('1', 10439, 'AC', 'A')) - self.assertEqual(cm.exception.response.status_code, 404) - self.assertEqual(str(cm.exception), 'Variant not present in seqr') - self._test_minimal_search_call(url_path='lookup', expected_search_body={ - 'variant_id': ['1', 10439, 'AC', 'A'], 'genome_version': 'GRCh38', 'data_type': 'SNV_INDEL', - }) - - # Test mitochondrial variant lookup - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/lookup', status=400) - with self.assertRaises(InvalidSearchException) as cm: - variant_lookup(self.user, ('M', 11018, 'G', 'T'), genome_version='37') - self.assertEqual(str(cm.exception), 'MITO variants are not available for GRCh37') - - @responses.activate - def test_sv_variant_lookup(self): - sv_families = Family.objects.filter(id__in=[2, 14]) - with self.assertRaises(InvalidSearchException) as cm: - sv_variant_lookup(self.user, 'suffix_140608_DUP', sv_families) - self.assertEqual(str(cm.exception), 'Sample type must be specified to look up a structural variant') - - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/lookup', status=200, json=GCNV_VARIANT4) - variants = sv_variant_lookup(self.user, 'suffix_140608_DUP', sv_families, sample_type='WES') - self.assertListEqual(variants, [GCNV_VARIANT4] + HAIL_BACKEND_VARIANTS) - self._test_minimal_search_call(url_path='lookup', call_offset=-2, expected_search_body={ - 'variant_id': 'suffix_140608_DUP', 'genome_version': 'GRCh38', 'data_type': 'SV_WES', - 'sample_data': ALL_AFFECTED_SAMPLE_DATA['SV_WES'] - }) - self._test_minimal_search_call(expected_search_body={ - 'genome_version': 'GRCh38', 'annotations': {'structural': ['DEL', 'gCNV_DEL']}, - 'padded_interval': {'chrom': '17', 'start': 38721781, 'end': 38735703, 'padding': 0.2}, - 'sample_data': {'SV_WGS': SV_WGS_SAMPLE_DATA}, 'frequencies': None, 'quality_filter': None, - 'sort': None, 'sort_metadata': None, 'num_results': 100, - }) - - # No second lookup call is made for non DELs/DUPs - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/lookup', status=200, json=SV_VARIANT2) - variants = sv_variant_lookup(self.user, 'cohort_2911.chr1.final_cleanup_INS_chr1_160', sv_families, sample_type='WGS') - self._test_minimal_search_call(url_path='lookup', expected_search_body={ - 'variant_id': 'cohort_2911.chr1.final_cleanup_INS_chr1_160', 'genome_version': 'GRCh38', 'data_type': 'SV_WGS', - 'sample_data': SV_WGS_SAMPLE_DATA - }) - self.assertListEqual(variants, [SV_VARIANT2]) - - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/lookup', status=404) - with self.assertRaises(HTTPError) as cm: - sv_variant_lookup(self.user, 'suffix_140608_DUP', sv_families, sample_type='WES') - self.assertEqual(cm.exception.response.status_code, 404) - self.assertEqual(str(cm.exception), 'Variant not present in seqr') - - @responses.activate - def test_get_single_variant(self): - family = self.families.filter(guid='F000002_2').first() - variant = get_single_variant(family, '2-103343353-GAGA-G', user=self.user) - self.assertDictEqual(variant, HAIL_BACKEND_VARIANTS[0]) - self._test_minimal_search_call( - variant_ids=[['2', 103343353, 'GAGA', 'G']], variant_keys=[], - num_results=1, sample_data={'SNV_INDEL': FAMILY_2_VARIANT_SAMPLE_DATA['SNV_INDEL']}) - - get_single_variant(family, 'prefix_19107_DEL', user=self.user) - self._test_minimal_search_call( - variant_ids=[], variant_keys=['prefix_19107_DEL'], - num_results=1, sample_data=EXPECTED_SAMPLE_DATA, omit_data_type='SNV_INDEL') - - get_single_variant(family, 'M-10195-C-A', user=self.user) - self._test_minimal_search_call( - variant_ids=[['M', 10195, 'C', 'A']], variant_keys=[], - num_results=1, sample_data=EXPECTED_MITO_SAMPLE_DATA) - - - get_single_variant(family, '2-103343353-GAGA-G', user=self.user) - self._test_minimal_search_call( - variant_ids=[['2', 103343353, 'GAGA', 'G']], variant_keys=[], - num_results=1, sample_data=FAMILY_2_VARIANT_SAMPLE_DATA) - - responses.add(responses.POST, f'{MOCK_ORIGIN}:5000/search', status=200, json={'results': [], 'total': 0}) - with self.assertRaises(InvalidSearchException) as cm: - get_single_variant(family, '10-10334333-A-G', user=self.user) - self.assertEqual(str(cm.exception), 'Variant 10-10334333-A-G not found') - - @responses.activate - def test_get_variants_for_variant_ids(self): - variant_ids = ['2-103343353-GAGA-G', '1-248367227-TC-T', 'prefix-938_DEL'] - get_variants_for_variant_ids(self.families, variant_ids, user=self.user) - expected_sample_data = {k: ALL_AFFECTED_SAMPLE_DATA[k] for k in ['SNV_INDEL', 'SV_WES']} - self._test_minimal_search_call( - variant_ids=[['2', 103343353, 'GAGA', 'G'], ['1', 248367227, 'TC', 'T']], - variant_keys=['prefix-938_DEL'], - num_results=3, sample_data=expected_sample_data) - - del expected_sample_data['SV_WES'] - get_variants_for_variant_ids(self.families, variant_ids, user=self.user, dataset_type='SNV_INDEL') - self._test_minimal_search_call( - variant_ids=[['2', 103343353, 'GAGA', 'G'], ['1', 248367227, 'TC', 'T']], - variant_keys=[], - num_results=2, sample_data=expected_sample_data) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index e068739aec..ae6b57e4c4 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -5,11 +5,11 @@ import mock from clickhouse_search.test_utils import VARIANT1, VARIANT2, VARIANT3, VARIANT4, PROJECT_2_VARIANT2, format_cached_variant, \ - GENE_COUNTS, VARIANT_LOOKUP_VARIANT, SV_VARIANT4, SV_VARIANT1 + GENE_COUNTS, VARIANT_LOOKUP_VARIANT, SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, SV_VARIANT1 from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets -from seqr.utils.search.utils import get_single_variant, get_variants_for_variant_ids, get_variant_query_gene_counts, \ - query_variants, variant_lookup, sv_variant_lookup, InvalidSearchException +from seqr.utils.search.utils import get_single_variant, get_variant_query_gene_counts, \ + query_variants, variant_lookup, InvalidSearchException from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin, PARSED_VARIANTS, PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT, GENE_FIELDS @@ -51,6 +51,7 @@ def assert_cached_results(self, expected_results, sort='xpos', cache_key=None): class SearchUtilsTests(SearchTestHelper): CACHED_VARIANTS = PARSED_VARIANTS + PARSED_VARIANTS + HAS_GENE_AGG = False def set_up(self): super(SearchUtilsTests, self).set_up() @@ -65,44 +66,43 @@ def set_up(self): self.search_samples = list(self.affected_search_samples) + list(self.non_affected_search_samples) def test_variant_lookup(self, mock_variant_lookup): - mock_variant_lookup.return_value = VARIANT_LOOKUP_VARIANT - variant = variant_lookup(self.user, ('1', 10439, 'AC', 'A'), genome_version='38') - self.assertDictEqual(variant, VARIANT_LOOKUP_VARIANT) - mock_variant_lookup.assert_called_with(self.user, ('1', 10439, 'AC', 'A'), 'SNV_INDEL', genome_version='38') - cache_key = "variant_lookup_results__('1', 10439, 'AC', 'A')__38__" - self.assert_cached_results(variant, cache_key=cache_key) + mock_variant_lookup.return_value = [VARIANT_LOOKUP_VARIANT] + variants = variant_lookup(self.user, '1-10439-AC-A', '38') + self.assertListEqual(variants, [VARIANT_LOOKUP_VARIANT]) + mock_variant_lookup.assert_called_with(self.user, ('1', 10439, 'AC', 'A'), 'SNV_INDEL', None, '38') + cache_key = "variant_lookup_results__1-10439-AC-A__38" + self.assert_cached_results(variants, cache_key=cache_key) mock_variant_lookup.reset_mock() - self.set_cache(variant) - cached_variant = variant_lookup(self.user, ('1', 10439, 'AC', 'A'), genome_version='38') - self.assertDictEqual(variant, cached_variant) + self.set_cache(variants) + cached_variant = variant_lookup(self.user, '1-10439-AC-A', '38') + self.assertListEqual(variants, cached_variant) mock_variant_lookup.assert_not_called() self.mock_redis.get.assert_called_with(cache_key) - def test_sv_variant_lookup(self, mock_sv_variant_lookup, mock_get_variants): - mock_sv_variant_lookup.return_value = SV_VARIANT4 - def _mock_get_variants(samples, search, user, previous_search_results, genome_version, **kwargs): - previous_search_results['all_results'] = [SV_VARIANT1] - mock_get_variants.side_effect = _mock_get_variants - variants = sv_variant_lookup(self.user, 'phase2_DEL_chr14_4640', self.families, genome_version='38', sample_type='WGS') - self.assertListEqual(variants, [SV_VARIANT4, SV_VARIANT1]) - mock_sv_variant_lookup.assert_called_with( - self.user, 'phase2_DEL_chr14_4640', 'SV_WGS', genome_version='38', samples=mock.ANY) - mock_get_variants.assert_called_with(mock.ANY, { - 'parsed_locus': {'padded_interval': {'chrom': '17', 'start': 38719997, 'end': 38737237, 'padding': 0.2}}, - 'annotations': {'structural': ['DEL', 'gCNV_DEL']}, - }, self.user, mock.ANY, '38') - cache_key = 'variant_lookup_results__phase2_DEL_chr14_4640__38__test_user' + self.set_cache(None) + mock_variant_lookup.reset_mock() + with self.assertRaises(InvalidSearchException) as cm: + variant_lookup(self.user, 'phase2_DEL_chr14_4640', '37') + self.assertEqual(str(cm.exception), 'SV variants are not available for GRCh37') + + with self.assertRaises(InvalidSearchException) as cm: + variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38') + self.assertEqual(str(cm.exception), 'Sample type must be specified to look up a structural variant') + + mock_variant_lookup.return_value = [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT] + variants = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS') + self.assertListEqual(variants, [SV_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT]) + mock_variant_lookup.assert_called_with( + self.user, 'phase2_DEL_chr14_4640', 'SV', 'WGS', '38') + cache_key = 'variant_lookup_results__phase2_DEL_chr14_4640__38' self.assert_cached_results(variants, cache_key=cache_key) - expected_samples = {s for s in self.search_samples if s.guid in SV_SAMPLES} - self.assertSetEqual(set(mock_get_variants.call_args.args[0]), expected_samples) - self.assertListEqual(list(mock_sv_variant_lookup.call_args.kwargs['samples']), []) - mock_sv_variant_lookup.reset_mock() + mock_variant_lookup.reset_mock() self.set_cache(variants) - cached_variant = sv_variant_lookup(self.user, 'phase2_DEL_chr14_4640', self.families, genome_version='38') + cached_variant = variant_lookup(self.user, 'phase2_DEL_chr14_4640', '38', sample_type='WGS') self.assertListEqual(variants, cached_variant) - mock_sv_variant_lookup.assert_not_called() + mock_variant_lookup.assert_not_called() self.mock_redis.get.assert_called_with(cache_key) def test_get_single_variant(self, mock_get_variants_for_ids): @@ -127,48 +127,14 @@ def test_get_single_variant(self, mock_get_variants_for_ids): get_single_variant(family, '10-10334333-A-G') self.assertEqual(str(cm.exception), 'Variant 10-10334333-A-G not found') - def _assert_expected_get_single_variant_call(self, mock_get_variants_for_ids, parsed_variant_id, expected_samples, **kwargs): - if isinstance(parsed_variant_id, str): - variants_id_map = {parsed_variant_id: None} - else: - variants_id_map = {'-'.join([str(v) for v in parsed_variant_id]): parsed_variant_id} + def _assert_expected_get_single_variant_call(self, mock_get_variants_for_ids, variant_id, expected_samples, **kwargs): + if not isinstance(variant_id, str): + variant_id = '-'.join([str(v) for v in variant_id]) mock_get_variants_for_ids.assert_called_with( - mock.ANY, '37', variants_id_map, self.user, + mock.ANY, '37', [variant_id], self.user, ) self.assertSetEqual(set(mock_get_variants_for_ids.call_args.args[0]), expected_samples) - def test_get_variants_for_variant_ids(self, mock_get_variants_for_ids): - variant_ids = ['2-103343353-GAGA-G', '1-248367227-TC-T', 'prefix-938_DEL', 'MT-10195-C-A'] - get_variants_for_variant_ids(self.families, variant_ids, user=self.user) - mock_get_variants_for_ids.assert_called_with(mock.ANY, '37', { - '2-103343353-GAGA-G': ('2', 103343353, 'GAGA', 'G'), - '1-248367227-TC-T': ('1', 248367227, 'TC', 'T'), - 'MT-10195-C-A': ('M', 10195, 'C', 'A'), - 'prefix-938_DEL': None, - }, self.user, user_email=None) - self.assertSetEqual(set(mock_get_variants_for_ids.call_args.args[0]), set(self.search_samples)) - - get_variants_for_variant_ids( - self.families, variant_ids, user=self.user, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) - mock_get_variants_for_ids.assert_called_with(mock.ANY, '37', { - '2-103343353-GAGA-G': ('2', 103343353, 'GAGA', 'G'), - '1-248367227-TC-T': ('1', 248367227, 'TC', 'T'), - 'MT-10195-C-A': ('M', 10195, 'C', 'A'), - }, self.user, user_email=None) - skipped_samples = ['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733'] - expected_samples = {s for s in self.search_samples if s.guid not in skipped_samples} - self.assertSetEqual(set(mock_get_variants_for_ids.call_args.args[0]), expected_samples) - - get_variants_for_variant_ids( - self.families, variant_ids[:3], user=self.user, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) - mock_get_variants_for_ids.assert_called_with(mock.ANY, '37', { - '2-103343353-GAGA-G': ('2', 103343353, 'GAGA', 'G'), - '1-248367227-TC-T': ('1', 248367227, 'TC', 'T'), - }, self.user, user_email=None) - skipped_samples.append('S000149_hg00733') - expected_samples = {s for s in self.search_samples if s.guid not in skipped_samples} - self.assertSetEqual(set(mock_get_variants_for_ids.call_args.args[0]), expected_samples) - @mock.patch('seqr.utils.search.utils.MAX_FAMILY_COUNTS', {'WES': 2, 'WGS': 1}) @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) def _test_invalid_search_params(self, search_func): @@ -317,14 +283,31 @@ def _test_expected_search_call(self, mock_get_variants, results_cache, search_fi rs_ids=None, variant_ids=None, parsed_variant_ids=None, inheritance_mode='de_novo', dataset_type=None, secondary_dataset_type=None, omitted_sample_guids=None, exclude_locations=False, exclude=None, annotations=None, annotations_secondary=None, single_gene_search=False, **kwargs): + genes = intervals = None + has_included_gene_search = has_gene_search and not exclude_locations + if has_gene_search: + genes = {'ENSG00000186092': mock.ANY, 'ENSG00000227232': mock.ANY} + intervals = [{'chrom': '2', 'start': 1234, 'end': 5678, 'offset': None}, {'chrom': '7', 'start': 100, 'end': 10100, 'offset': 0.1}] + if single_gene_search: + genes = {'ENSG00000227232': mock.ANY} + intervals = intervals[2:3] + expected_search = { 'inheritance_mode': inheritance_mode, 'inheritance_filter': {}, - 'parsed_locus': mock.ANY, 'skipped_samples': mock.ANY, - 'dataset_type': mock.ANY, + 'dataset_type': None if has_included_gene_search else dataset_type, 'secondary_dataset_type': secondary_dataset_type, + 'exclude_locations': exclude_locations, + 'genes': genes, + 'intervals': intervals, } + if not genes: + expected_search.update({ + 'parsed_variant_ids': parsed_variant_ids, + 'rs_ids': rs_ids, + 'variant_ids': variant_ids, + }) expected_search.update({field: self.search_model.search[field] for field in search_fields or []}) if exclude: expected_search['exclude'] = exclude @@ -336,19 +319,14 @@ def _test_expected_search_call(self, mock_get_variants, results_cache, search_fi mock_get_variants.assert_called_with(mock.ANY, expected_search, self.user, results_cache, '37', **kwargs) self._assert_expected_search_samples(mock_get_variants, omitted_sample_guids, has_gene_search and not exclude_locations) - gene_ids = intervals = None - has_included_gene_search = has_gene_search and not exclude_locations - if has_gene_search: - gene_ids = ['ENSG00000186092', 'ENSG00000227232'] - intervals = [['2', 1234, 5678], ['7', 1, 11100], ['1', 14404, 29570], ['1', 65419, 71585]] - if single_gene_search: - gene_ids = gene_ids[1:] - intervals = intervals[2:3] - self._assert_expected_search_locus( - mock_get_variants.call_args.args[1], dataset_type='MITO_missing' if has_included_gene_search else dataset_type, - gene_ids=gene_ids, intervals=intervals, rs_ids=rs_ids, variant_ids=variant_ids, - parsed_variant_ids=parsed_variant_ids, exclude_locations=exclude_locations, - ) + if genes: + parsed_genes = mock_get_variants.call_args.args[1]['genes'] + for gene in parsed_genes.values(): + self.assertSetEqual(set(gene.keys()), {'id', *GENE_FIELDS}) + self.assertEqual(parsed_genes['ENSG00000227232']['geneSymbol'], 'WASH7P') + if len(genes) > 1: + self.assertEqual(parsed_genes['ENSG00000186092']['geneSymbol'], 'OR4F5') + def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guids, has_gene_search): searched_samples = self.affected_search_samples @@ -362,12 +340,6 @@ def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guid self.assertSetEqual(set(mock_get_variants.call_args.args[1]['skipped_samples']), set(non_affected_search_samples)) - def _assert_expected_search_locus(self, search_body, dataset_type, **parsed_locus): - self.maxDiff = None - self.assertDictEqual(search_body['parsed_locus'], parsed_locus) - self.assertEqual(search_body['dataset_type'], dataset_type) - - def test_query_variants(self, mock_get_variants): def _mock_get_variants(families, search, user, previous_search_results, genome_version, **kwargs): previous_search_results['all_results'] = PARSED_VARIANTS @@ -431,14 +403,14 @@ def _mock_get_variants(families, search, user, previous_search_results, genome_v query_variants(self.results_model, user=self.user) self._test_expected_search_call( mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - has_gene_search=True, + has_gene_search=True, omitted_sample_guids=MITO_SAMPLES, ) self.search_model.search['locus']['rawItems'] = 'WASH7P' query_variants(self.results_model, user=self.user) self._test_expected_search_call( mock_get_variants, results_cache, sort='xpos', page=1, num_results=100, skip_genotype_filter=False, - has_gene_search=True, single_gene_search=True, + has_gene_search=True, single_gene_search=True, omitted_sample_guids=MITO_SAMPLES, ) del self.search_model.search['locus'] @@ -524,19 +496,13 @@ def _mock_get_gene_counts(families, search, user, previous_search_results, genom self.assertDictEqual(gene_counts, GENE_COUNTS) results_cache = {'all_results': self.GENE_AGG_ALL_RESULTS} if hasattr(self, 'GENE_AGG_ALL_RESULTS') else {'gene_aggs': gene_counts} self.assert_cached_results(results_cache) + kwargs = dict(sort=None, num_results=100) + if self.HAS_GENE_AGG: + kwargs['gene_agg'] = True self._test_expected_search_call( - mock_get_variants, results_cache, sort=None, num_results=100, gene_agg=True, + mock_get_variants, results_cache, **kwargs, ) - def test_cached_get_variant_query_gene_counts(self): - cached_gene_counts = { - 'ENSG00000135953': {'total': 5, 'families': {'F000003_3': 2, 'F000002_2': 1, 'F000011_11': 4}}, - 'ENSG00000228198': {'total': 5, 'families': {'F000003_3': 4, 'F000002_2': 1, 'F000011_11': 4}} - } - self.set_cache({'total_results': 5, 'gene_aggs': cached_gene_counts}) - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, cached_gene_counts) - @mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') @mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') @@ -544,55 +510,24 @@ class ElasticsearchSearchUtilsTests(TestCase, SearchUtilsTests): databases = ['default', 'reference_data'] fixtures = ['users', '1kg_project', 'reference_data'] + HAS_GENE_AGG = True + def setUp(self): self.set_up() - def _assert_expected_search_locus(self, search_body, dataset_type, gene_ids=None, intervals=None, **kwargs): - genes = None - if gene_ids: - genes = { - gene_id: mock.ANY for gene_id in gene_ids - } - intervals = [ - {'chrom': '2', 'start': 1234, 'end': 5678, 'offset': None}, - {'chrom': '7', 'start': 100, 'end': 10100, 'offset': 0.1}, - ] if len(gene_ids) > 1 else [] - - dataset_type = None if dataset_type == 'MITO_missing' else dataset_type - super()._assert_expected_search_locus( - search_body, dataset_type, genes=genes, intervals=intervals, **kwargs, - ) - - if gene_ids: - parsed_genes = search_body['parsed_locus']['genes'] - for gene in parsed_genes.values(): - self.assertSetEqual(set(gene.keys()), {'id', *GENE_FIELDS}) - self.assertEqual(parsed_genes['ENSG00000227232']['geneSymbol'], 'WASH7P') - if len(gene_ids) > 1: - self.assertEqual(parsed_genes['ENSG00000186092']['geneSymbol'], 'OR4F5') - def _assert_expected_search_samples(self, mock_get_variants, omitted_sample_guids, has_gene_search): return super()._assert_expected_search_samples(mock_get_variants, omitted_sample_guids, False) def test_variant_lookup(self, *args, **kwargs): - with self.assertRaises(InvalidSearchException) as cm: + with self.assertRaises(ValueError) as cm: super().test_variant_lookup(mock.MagicMock()) - self.assertEqual(str(cm.exception), 'Lookup is disabled') - - def test_sv_variant_lookup(self, *args, **kwargs): - with self.assertRaises(InvalidSearchException) as cm: - super().test_sv_variant_lookup(mock.MagicMock(), mock.MagicMock()) - self.assertEqual(str(cm.exception), 'Lookup is disabled') + self.assertEqual(str(cm.exception), 'variant_lookup is disabled without the clickhouse backend') @mock.patch('seqr.utils.search.utils.get_es_variants_for_variant_ids') def test_get_single_variant(self, mock_get_variants_for_ids): mock_get_variants_for_ids.return_value = [PARSED_VARIANTS[0]] super(ElasticsearchSearchUtilsTests, self).test_get_single_variant(mock_get_variants_for_ids) - @mock.patch('seqr.utils.search.utils.get_es_variants_for_variant_ids') - def test_get_variants_for_variant_ids(self, mock_get_variants_for_ids): - super(ElasticsearchSearchUtilsTests, self).test_get_variants_for_variant_ids(mock_get_variants_for_ids) - @mock.patch('seqr.utils.search.utils.get_es_variants') def test_query_variants(self, mock_get_variants): super(ElasticsearchSearchUtilsTests, self).test_query_variants(mock_get_variants) @@ -621,7 +556,13 @@ def test_get_variant_query_gene_counts(self, mock_get_variants): super(ElasticsearchSearchUtilsTests, self).test_get_variant_query_gene_counts(mock_get_variants) def test_cached_get_variant_query_gene_counts(self): - super(ElasticsearchSearchUtilsTests, self).test_cached_get_variant_query_gene_counts() + cached_gene_counts = { + 'ENSG00000135953': {'total': 5, 'families': {'F000003_3': 2, 'F000002_2': 1, 'F000011_11': 4}}, + 'ENSG00000228198': {'total': 5, 'families': {'F000003_3': 4, 'F000002_2': 1, 'F000011_11': 4}} + } + self.set_cache({'total_results': 5, 'gene_aggs': cached_gene_counts}) + gene_counts = get_variant_query_gene_counts(self.results_model, self.user) + self.assertDictEqual(gene_counts, cached_gene_counts) self.set_cache({'all_results': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT, 'total_results': 2}) gene_counts = get_variant_query_gene_counts(self.results_model, self.user) @@ -654,58 +595,6 @@ def _test_invalid_no_location_search_params(self): pass -@mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') -class HailSearchUtilsTests(TestCase, SearchUtilsTests): - databases = ['default', 'reference_data'] - fixtures = ['users', '1kg_project', 'reference_data'] - - def setUp(self): - self.set_up() - - def _assert_expected_search_locus(self, *args, gene_ids=None, exclude_locations=None, parsed_variant_ids=None, variant_ids=None, **kwargs): - super()._assert_expected_search_locus( - *args, gene_ids=None if exclude_locations else gene_ids, variant_ids=parsed_variant_ids, - exclude_intervals=exclude_locations, **kwargs, - ) - - @mock.patch('seqr.utils.search.utils.hail_variant_lookup') - def test_variant_lookup(self, mock_variant_lookup): - super(HailSearchUtilsTests, self).test_variant_lookup(mock_variant_lookup) - - @mock.patch('seqr.utils.search.utils.get_hail_variants') - @mock.patch('seqr.utils.search.utils.hail_variant_lookup') - def test_sv_variant_lookup(self, mock_sv_variant_lookup, mock_get_variants): - super(HailSearchUtilsTests, self).test_sv_variant_lookup(mock_sv_variant_lookup, mock_get_variants) - - @mock.patch('seqr.utils.search.utils.get_hail_variants_for_variant_ids') - def test_get_single_variant(self, mock_call): - mock_call.return_value = [PARSED_VARIANTS[0]] - super(HailSearchUtilsTests, self).test_get_single_variant(mock_call) - - @mock.patch('seqr.utils.search.utils.get_hail_variants_for_variant_ids') - def test_get_variants_for_variant_ids(self, mock_call): - super(HailSearchUtilsTests, self).test_get_variants_for_variant_ids(mock_call) - - @mock.patch('seqr.utils.search.utils.get_hail_variants') - def test_query_variants(self, mock_call): - super(HailSearchUtilsTests, self).test_query_variants(mock_call) - - @mock.patch('seqr.utils.search.utils.get_hail_variants') - def test_get_variant_query_gene_counts(self, mock_call): - super(HailSearchUtilsTests, self).test_get_variant_query_gene_counts(mock_call) - - def test_cached_get_variant_query_gene_counts(self): - super(HailSearchUtilsTests, self).test_cached_get_variant_query_gene_counts() - - self.set_cache({'all_results': [PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT] + [SV_VARIANT1], 'total_results': 2}) - gene_counts = get_variant_query_gene_counts(self.results_model, self.user) - self.assertDictEqual(gene_counts, { - 'ENSG00000135953': {'total': 2, 'families': {'F000003_3': 2, 'F000011_11': 2}}, - 'ENSG00000228198': {'total': 2, 'families': {'F000003_3': 2, 'F000011_11': 2}}, - 'ENSG00000171621': {'total': 1, 'families': {'F000014_14': 1}}, - }) - - @mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', 'testhost') class ClickhouseSearchUtilsTests(DifferentDbTransactionSupportMixin, TestCase, SearchUtilsTests): databases = '__all__' @@ -724,20 +613,6 @@ def _assert_expected_cached_variants(self, variants, num_results): self.PARSED_CACHED_VARIANTS[:num_results], ) - def _assert_expected_search_locus(self, *args, gene_ids=None, intervals=None, exclude_locations=None, parsed_variant_ids=None, variant_ids=None, **kwargs): - gene_ids = None if exclude_locations else gene_ids - gene_intervals = None - if gene_ids: - if len(gene_ids) > 1: - gene_intervals = {2: intervals[2], 7: intervals[3]} - else: - gene_intervals = {2: intervals[0]} - intervals = intervals[:2] if len(gene_ids) > 1 else None - super()._assert_expected_search_locus( - *args, gene_ids=gene_ids, gene_intervals=gene_intervals, intervals=intervals, - variant_ids=parsed_variant_ids, exclude_intervals=exclude_locations, **kwargs, - ) - @mock.patch('seqr.utils.search.utils.get_clickhouse_variant_by_id') def test_get_single_variant(self, mock_call): mock_call.return_value = PARSED_VARIANTS[0] @@ -747,11 +622,6 @@ def _assert_expected_get_single_variant_call(self, mock_call, variant_id, expect mock_call.assert_called_with(variant_id, mock.ANY, '37', dataset_type) self.assertSetEqual(set(mock_call.call_args.args[1]), expected_samples) - @mock.patch('seqr.utils.search.utils.get_clickhouse_variants') - def test_get_variants_for_variant_ids(self, mock_call): - with self.assertRaises(NotImplementedError): - super().test_get_variants_for_variant_ids(mock_call) - @mock.patch('seqr.utils.search.utils.get_clickhouse_variants') def test_query_variants(self, mock_call): super().test_query_variants(mock_call) @@ -785,8 +655,6 @@ def test_get_variant_query_gene_counts(self, mock_call): super().test_get_variant_query_gene_counts(mock_call) def test_cached_get_variant_query_gene_counts(self): - super().test_cached_get_variant_query_gene_counts() - self.set_cache({'all_results': self.CACHED_VARIANTS + [SV_VARIANT1], 'total_results': 5}) gene_counts = get_variant_query_gene_counts(self.results_model, self.user) self.assertDictEqual(gene_counts, { @@ -799,8 +667,3 @@ def test_cached_get_variant_query_gene_counts(self): @mock.patch('seqr.utils.search.utils.clickhouse_variant_lookup') def test_variant_lookup(self, mock_call): super().test_variant_lookup(mock_call) - - @mock.patch('seqr.utils.search.utils.get_clickhouse_variants') - @mock.patch('seqr.utils.search.utils.clickhouse_variant_lookup') - def test_sv_variant_lookup(self, mock_sv_variant_lookup, mock_get_variants): - super().test_sv_variant_lookup(mock_sv_variant_lookup, mock_get_variants) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 93cee91617..2122c99e7c 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -4,7 +4,7 @@ from django.db.models import Count from pyliftover.liftover import LiftOver -from clickhouse_search.search import clickhouse_backend_enabled, get_clickhouse_variants, format_clickhouse_results, \ +from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, \ get_clickhouse_cache_results, clickhouse_variant_lookup, get_clickhouse_variant_by_id from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 from seqr.models import Sample, Individual, Project @@ -13,11 +13,9 @@ from seqr.utils.search.constants import XPOS_SORT_KEY, PRIORITIZED_GENE_SORT, RECESSIVE, COMPOUND_HET, \ MAX_NO_LOCATION_COMP_HET_FAMILIES, SV_ANNOTATION_TYPES, ALL_DATA_TYPES, MAX_EXPORT_VARIANTS, X_LINKED_RECESSIVE, \ MAX_VARIANTS -from seqr.utils.search.elasticsearch.es_utils import ping_elasticsearch, delete_es_index, get_elasticsearch_status, \ +from seqr.utils.search.elasticsearch.es_utils import ping_elasticsearch, \ get_es_variants, get_es_variants_for_variant_ids, process_es_previously_loaded_results, process_es_previously_loaded_gene_aggs, \ es_backend_enabled, ping_kibana, ES_EXCEPTION_ERROR_MAP, ES_EXCEPTION_MESSAGE_MAP, ES_ERROR_LOG_EXCEPTIONS -from seqr.utils.search.hail_search_utils import get_hail_variants, get_hail_variants_for_variant_ids, ping_hail_backend, \ - hail_variant_lookup from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.xpos_utils import get_xpos, format_chrom, MIN_POS, MAX_POS @@ -52,36 +50,36 @@ class InvalidSearchException(Exception): DATASET_TYPES_LOOKUP[DATASET_TYPE_NO_MITO] = [Sample.DATASET_TYPE_VARIANT_CALLS, Sample.DATASET_TYPE_SV_CALLS] -def _raise_search_error(error): +def es_only(func): def _wrapped(*args, **kwargs): - raise InvalidSearchException(error) + if not es_backend_enabled(): + raise ValueError(f'{func.__name__} is disabled without the elasticsearch backend') + return func(*args, **kwargs) return _wrapped -def _raise_clickhouse_not_implemented(*args, **kwargs): - raise NotImplementedError('Clickhouse backend is not implemented for this function.') +def clickhouse_only(func): + def _wrapped(*args, **kwargs): + if es_backend_enabled(): + raise ValueError(f'{func.__name__} is disabled without the clickhouse backend') + return func(*args, **kwargs) + return _wrapped -def backend_specific_call(es_func, hail_backend_func, clickhouse_func): +def backend_specific_call(es_func, clickhouse_func): if es_backend_enabled(): return es_func - elif clickhouse_backend_enabled(): - return clickhouse_func else: - return hail_backend_func + return clickhouse_func def ping_search_backend(): # Clickhouse backend does not need special uptime testing, will be checked with the other database connection pings - backend_specific_call(ping_elasticsearch, ping_hail_backend, lambda: None)() + backend_specific_call(ping_elasticsearch, lambda: None)() def ping_search_backend_admin(): - backend_specific_call(ping_kibana, lambda: True, lambda: True)() - - -def get_search_backend_status(): - return backend_specific_call(get_elasticsearch_status, _raise_search_error('Elasticsearch is disabled'), _raise_search_error('Elasticsearch is disabled'))() + backend_specific_call(ping_kibana, lambda: True)() def _get_filtered_search_samples(search_filter, active_only=True): @@ -124,157 +122,109 @@ def _get_search_genome_version(families): return next(iter(project_versions.keys())) -def delete_search_backend_data(data_id): - active_samples = Sample.objects.filter(is_active=True, elasticsearch_index=data_id) - if active_samples: - projects = set(active_samples.values_list('individual__family__project__name', flat=True)) - raise InvalidSearchException(f'"{data_id}" is still used by: {", ".join(projects)}') - - return backend_specific_call( - delete_es_index, _raise_search_error('Deleting indices is disabled without the elasticsearch backend'), - _raise_search_error('Deleting indices is disabled without the elasticsearch backend'), - )(data_id) - - def get_single_variant(family, variant_id, user=None): parsed_variant_id = parse_variant_id(variant_id) dataset_type = _variant_ids_dataset_type([parsed_variant_id]) samples = _get_families_search_data([family], dataset_type, sample_filter={'individual__family_id': family.id}) variant = backend_specific_call( - _process_ids_search(get_es_variants_for_variant_ids), - _process_ids_search(get_hail_variants_for_variant_ids), + _get_es_variant_by_id, _get_clickhouse_variant_by_id, - )(parsed_variant_id, variant_id, samples, family.project.genome_version, dataset_type, user) + )(parsed_variant_id, variant_id, samples, family.project.genome_version, dataset_type=dataset_type, user=user) if not variant: raise InvalidSearchException('Variant {} not found'.format(variant_id)) return variant -def _process_ids_search(search_func): - def _search(parsed_variant_id, variant_id, samples, genome_version, dataset_type, user): - variants = search_func(samples, genome_version, {variant_id: parsed_variant_id}, user) - return variants[0] if variants else None - return _search +def _get_es_variant_by_id(parsed_variant_id, variant_id, samples, genome_version, user=None, **kwargs): + variants = get_es_variants_for_variant_ids(samples, genome_version, [variant_id], user) + return variants[0] if variants else None -def _get_clickhouse_variant_by_id(parsed_variant_id, variant_id, samples, genome_version, dataset_type, user): +def _get_clickhouse_variant_by_id(parsed_variant_id, variant_id, samples, genome_version, dataset_type=None, **kwargs): return get_clickhouse_variant_by_id( parsed_variant_id or variant_id, samples, genome_version, DATASET_TYPES_LOOKUP[dataset_type][0], ) -def get_variants_for_variant_ids(families, variant_ids, dataset_type=None, user=None, user_email=None, **kwargs): - parsed_variant_ids = {} - for variant_id in variant_ids: - parsed_variant_ids[variant_id] = parse_variant_id(variant_id) - - if dataset_type: - parsed_variant_ids = { - k: v for k, v in parsed_variant_ids.items() - if (dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and v) or - (dataset_type != Sample.DATASET_TYPE_VARIANT_CALLS and not v) - } - dataset_type = _variant_ids_dataset_type(parsed_variant_ids.values()) - - return backend_specific_call(get_es_variants_for_variant_ids, get_hail_variants_for_variant_ids, _raise_clickhouse_not_implemented)( - _get_families_search_data(families, dataset_type=dataset_type), _get_search_genome_version(families), - parsed_variant_ids, user, user_email=user_email, - ) +@clickhouse_only +def variant_lookup(user, variant_id, genome_version, sample_type=None): + cache_key = f'variant_lookup_results__{variant_id}__{genome_version}' + variants = safe_redis_get_json(cache_key) + if variants: + return variants + parsed_variant_id = parse_variant_id(variant_id) + dataset_type = DATASET_TYPES_LOOKUP[_variant_ids_dataset_type([parsed_variant_id])][0] + _validate_dataset_type_genome_version(dataset_type, sample_type, genome_version) -def _variant_lookup(lookup_func, user, variant_id, dataset_type, genome_version=None, cache_key_suffix='', **kwargs): - genome_version = genome_version or GENOME_VERSION_GRCh38 - _validate_dataset_type_genome_version(dataset_type, genome_version) - cache_key = f'variant_lookup_results__{variant_id}__{genome_version}__{cache_key_suffix}' - variant = safe_redis_get_json(cache_key) - if variant: - return variant + variants = clickhouse_variant_lookup(user, parsed_variant_id or variant_id, dataset_type, sample_type, genome_version) - variant = lookup_func(user, variant_id, dataset_type, genome_version=genome_version, **kwargs) - safe_redis_set_json(cache_key, variant, expire=timedelta(weeks=2)) - return variant + safe_redis_set_json(cache_key, variants, expire=timedelta(weeks=2)) + return variants -def _validate_dataset_type_genome_version(dataset_type, genome_version): +def _validate_dataset_type_genome_version(dataset_type, sample_type, genome_version): if genome_version == GENOME_VERSION_GRCh37 and dataset_type != Sample.DATASET_TYPE_VARIANT_CALLS: raise InvalidSearchException(f'{dataset_type} variants are not available for GRCh37') + if dataset_type == Sample.DATASET_TYPE_SV_CALLS and not sample_type: + raise InvalidSearchException('Sample type must be specified to look up a structural variant') -def variant_lookup(user, parsed_variant_id, **kwargs): - dataset_type = DATASET_TYPES_LOOKUP[_variant_ids_dataset_type([parsed_variant_id])][0] - lookup_func = backend_specific_call(_raise_search_error('Lookup is disabled'), hail_variant_lookup, clickhouse_variant_lookup) - return _variant_lookup(lookup_func, user, parsed_variant_id, **kwargs, dataset_type=dataset_type) - +def _get_search_cache_key(search_model, sort=None): + return 'search_results__{}__{}'.format(search_model.guid, sort or XPOS_SORT_KEY) -def sv_variant_lookup(user, variant_id, families, **kwargs): - _get_search_genome_version(families) - samples = _get_families_search_data(families, dataset_type=Sample.DATASET_TYPE_SV_CALLS) - return _variant_lookup( - _sv_variant_lookup, user, variant_id, **kwargs, samples=samples, cache_key_suffix=user, - dataset_type=Sample.DATASET_TYPE_SV_CALLS, - ) +def _get_any_sort_cached_results(search_model): + cache_key = _get_search_cache_key(search_model, sort='*') + return safe_redis_get_wildcard_json(cache_key) -def _sv_variant_lookup(user, variant_id, dataset_type, samples, genome_version=None, sample_type=None, **kwargs): - if not sample_type: - raise InvalidSearchException('Sample type must be specified to look up a structural variant') +def _get_cached_search_results(search_model, sort=None): + cache_key = _get_search_cache_key(search_model, sort=sort) + return safe_redis_get_json(cache_key) or {} - data_type = f'{dataset_type}_{sample_type}' - - lookup_samples = samples.filter(sample_type=sample_type) - lookup_func = backend_specific_call(_raise_search_error('Lookup is disabled'), hail_variant_lookup, clickhouse_variant_lookup) - variant = lookup_func(user, variant_id, data_type, samples=lookup_samples, genome_version=genome_version, **kwargs) - variants = [variant] - - if variant['svType'] in {'DEL', 'DUP'}: - samples = samples.exclude(sample_type=sample_type) - search = { - 'parsed_locus': { - 'padded_interval': {'chrom': variant['chrom'], 'start': variant['pos'], 'end': variant['end'], 'padding': 0.2}, - }, - 'annotations': {'structural': [variant['svType'], f"gCNV_{variant['svType']}"]}, - **kwargs, - } - results = {} - _execute_search(samples, search, user, previous_search_results=results, genome_version=genome_version) - variants += results['all_results'] - return variants +def _validate_export_variant_count(total_variants): + if total_variants > MAX_EXPORT_VARIANTS: + raise InvalidSearchException(f'Unable to export more than {MAX_EXPORT_VARIANTS} variants ({total_variants} requested)') -def _get_search_cache_key(search_model, sort=None): - return 'search_results__{}__{}'.format(search_model.guid, sort or XPOS_SORT_KEY) +def _get_elasticsearch_previous_search_results(search_model, sort, page, num_results, load_all, **kwargs): + previous_search_results = _get_cached_search_results(search_model, sort=sort) + start_index, end_index, num_results = _get_result_range(page, num_results, previous_search_results.get('total_results'), load_all) + cached_page = None + loaded_results = previous_search_results.get('all_results') or [] + if len(loaded_results) >= end_index: + cached_page = loaded_results[start_index:end_index] -def _process_clickhouse_unsorted_cached_results(cache_key, sort, family_guid): - unsorted_results = safe_redis_get_wildcard_json(cache_key.replace(sort or 'xpos', '*')) - if not unsorted_results: - return None - results = get_clickhouse_cache_results(unsorted_results['all_results'], sort, family_guid) - safe_redis_set_json(cache_key, results, expire=timedelta(weeks=2)) - return results + if not cached_page: + cached_page = process_es_previously_loaded_results(previous_search_results, start_index, end_index) + return previous_search_results, cached_page, num_results -def _get_cached_search_results(search_model, sort=None, family_guid=None): - cache_key = _get_search_cache_key(search_model, sort=sort) - results = safe_redis_get_json(cache_key) - if not results: - results = backend_specific_call( - lambda *args: None, lambda *args: None, _process_clickhouse_unsorted_cached_results, - )(cache_key, sort, family_guid) - return results or {} +def _get_clickhouse_previous_search_results(search_model, sort, page, num_results, load_all, genome_version=None): + previous_search_results = _get_cached_search_results(search_model, sort=sort) + if not previous_search_results: + unsorted_results = _get_any_sort_cached_results(search_model) + if unsorted_results: + previous_search_results = get_clickhouse_cache_results( + unsorted_results['all_results'], sort, family_guid=search_model.families.first().guid, + ) + cache_key = _get_search_cache_key(search_model, sort=sort) + safe_redis_set_json(cache_key, previous_search_results, expire=timedelta(weeks=2)) -def _validate_export_variant_count(total_variants): - if total_variants > MAX_EXPORT_VARIANTS: - raise InvalidSearchException(f'Unable to export more than {MAX_EXPORT_VARIANTS} variants ({total_variants} requested)') + start_index, end_index, num_results = _get_result_range(page, num_results, previous_search_results.get('total_results'), load_all) + cached_page = None + loaded_results = previous_search_results.get('all_results') or [] + if len(loaded_results) >= end_index: + cached_page = format_clickhouse_results(loaded_results[start_index:end_index], genome_version) + return previous_search_results, cached_page, num_results -def query_variants(search_model, sort=XPOS_SORT_KEY, skip_genotype_filter=False, load_all=False, user=None, page=1, num_results=100): - previous_search_results = _get_cached_search_results(search_model, sort=sort, family_guid=search_model.families.first().guid) - total_results = previous_search_results.get('total_results') +def _get_result_range(page, num_results, total_results, load_all): if load_all: num_results = total_results or MAX_EXPORT_VARIANTS _validate_export_variant_count(num_results) @@ -283,25 +233,21 @@ def query_variants(search_model, sort=XPOS_SORT_KEY, skip_genotype_filter=False, if total_results is not None: end_index = min(end_index, total_results) - genome_version = _get_search_genome_version(search_model.families.all()) - loaded_results = previous_search_results.get('all_results') or [] - if len(loaded_results) >= end_index: - results_page = backend_specific_call( - lambda results, genome_version: results, lambda results, genome_version: results, format_clickhouse_results, - )(loaded_results[start_index:end_index], genome_version) - return results_page, total_results - - previously_loaded_results = backend_specific_call( - process_es_previously_loaded_results, - lambda *args: None, # Other backends need no additional parsing - lambda *args: None, - )(previous_search_results, start_index, end_index) - if previously_loaded_results is not None: - return previously_loaded_results, total_results - if end_index > MAX_VARIANTS: raise InvalidSearchException(f'Unable to load more than {MAX_VARIANTS} variants ({end_index} requested)') + return start_index, end_index, num_results + + +def query_variants(search_model, sort=XPOS_SORT_KEY, skip_genotype_filter=False, load_all=False, user=None, page=1, num_results=100): + genome_version = _get_search_genome_version(search_model.families.all()) + previous_search_results, cached_page, num_results = backend_specific_call( + _get_elasticsearch_previous_search_results, + _get_clickhouse_previous_search_results, + )(search_model, sort, page, num_results, load_all, genome_version=genome_version) + if cached_page is not None: + return cached_page, previous_search_results.get('total_results') + variants, total_results = _query_variants( search_model, user, previous_search_results, genome_version, sort=sort, page=page, num_results=num_results, skip_genotype_filter=skip_genotype_filter) @@ -324,18 +270,19 @@ def _query_variants(search_model, user, previous_search_results, genome_version, if locus and exclude_locations: raise InvalidSearchException('Cannot specify both Location and Excluded Genes/Intervals') - rs_ids = None variant_ids = None - parsed_variant_ids = None + parsed_search = {**search} genes, intervals, invalid_items = parse_locus_list_items(locus or exclude, genome_version=genome_version, additional_model_fields=['id']) if invalid_items: raise InvalidSearchException('Invalid genes/intervals: {}'.format(', '.join(invalid_items))) + parsed_search.update({'genes': genes, 'intervals': intervals, 'exclude_locations': exclude_locations}) if not (genes or intervals): rs_ids, variant_ids, parsed_variant_ids, invalid_items = _parse_variant_items(locus) if invalid_items: raise InvalidSearchException('Invalid variants: {}'.format(', '.join(invalid_items))) if rs_ids and variant_ids: raise InvalidSearchException('Invalid variant notation: found both variant IDs and rsIDs') + parsed_search.update({'rs_ids': rs_ids, 'variant_ids': variant_ids, 'parsed_variant_ids': parsed_variant_ids}) if variant_ids: num_results = len(variant_ids) @@ -346,19 +293,12 @@ def _query_variants(search_model, user, previous_search_results, genome_version, if duplicates: raise InvalidSearchException(f'ClinVar pathogenicity {", ".join(sorted(duplicates))} is both included and excluded') - parsed_search = { - 'parsed_locus': backend_specific_call( - lambda genome_version, **kwargs: kwargs, _parse_locus_intervals, _parse_locus_gene_intervals, - )(genome_version, genes=genes, intervals=intervals, rs_ids=rs_ids, variant_ids=variant_ids, - parsed_variant_ids=parsed_variant_ids, exclude_locations=exclude_locations), - } - parsed_search.update(search) for annotation_key in ['annotations', 'annotations_secondary']: if parsed_search.get(annotation_key): parsed_search[annotation_key] = {k: v for k, v in parsed_search[annotation_key].items() if v} - dataset_type, secondary_dataset_type, lookup_dataset_type = _search_dataset_type(parsed_search) - parsed_search.update({'dataset_type': dataset_type, 'secondary_dataset_type': secondary_dataset_type}) + dataset_type, secondary_dataset_type, lookup_dataset_type = _search_dataset_type(parsed_search, genome_version) + parsed_search.update({'dataset_type': None if dataset_type == DATASET_TYPE_NO_MITO else dataset_type, 'secondary_dataset_type': secondary_dataset_type}) search_dataset_type = None if dataset_type and dataset_type != ALL_DATA_TYPES: if secondary_dataset_type is None or secondary_dataset_type == dataset_type: @@ -372,7 +312,7 @@ def _query_variants(search_model, user, previous_search_results, genome_version, _validate_search(parsed_search, samples, previous_search_results) - variant_results = _execute_search( + variant_results = backend_specific_call(get_es_variants, get_clickhouse_variants)( samples, parsed_search, user, previous_search_results, genome_version, sort=sort, num_results=num_results, **kwargs, ) @@ -383,54 +323,51 @@ def _query_variants(search_model, user, previous_search_results, genome_version, return variant_results, previous_search_results.get('total_results') -def _execute_search(samples, parsed_search, user, previous_search_results, genome_version, **kwargs): - return backend_specific_call(get_es_variants, get_hail_variants, get_clickhouse_variants)( - samples, parsed_search, user, previous_search_results, genome_version, **kwargs, - ) +def get_variant_query_gene_counts(search_model, user): + return backend_specific_call( + _get_es_variant_query_gene_counts, + _get_clickhouse_variant_query_gene_counts, + )(search_model, user) -def get_variant_query_gene_counts(search_model, user): +def _get_es_variant_query_gene_counts(search_model, user): previous_search_results = _get_cached_search_results(search_model) if previous_search_results.get('gene_aggs'): return previous_search_results['gene_aggs'] if len(previous_search_results.get('all_results', [])) == previous_search_results.get('total_results'): - return _get_gene_aggs_for_cached_variants(previous_search_results) + return _get_gene_aggs_for_cached_variants( + previous_search_results['all_results'], + lambda v: next(( + [gene_id] for gene_id, transcripts in v['transcripts'].items() + if any(t['transcriptId'] == v['mainTranscriptId'] for t in transcripts) + ), []) if v['mainTranscriptId'] else [], + ) - previously_loaded_results = backend_specific_call( - process_es_previously_loaded_gene_aggs, - lambda *args: None, # Other backends need no additional parsing - lambda *args: None, - )(previous_search_results) + previously_loaded_results = process_es_previously_loaded_gene_aggs(previous_search_results) if previously_loaded_results is not None: return previously_loaded_results genome_version = _get_search_genome_version(search_model.families.all()) gene_counts, _ = _query_variants(search_model, user, previous_search_results, genome_version, gene_agg=True) - gene_counts = backend_specific_call( - lambda *args: None, lambda *args: None, _get_gene_aggs_for_cached_variants, - )(previous_search_results) or gene_counts return gene_counts -def _get_gene_aggs_for_cached_variants(previous_search_results): +def _get_clickhouse_variant_query_gene_counts(search_model, user): + previous_search_results = _get_any_sort_cached_results(search_model) or {} + if len(previous_search_results.get('all_results', [])) != previous_search_results.get('total_results'): + genome_version = _get_search_genome_version(search_model.families.all()) + _query_variants(search_model, user, previous_search_results, genome_version) + + return _get_gene_aggs_for_cached_variants([ + v for variants in previous_search_results['all_results'] for v in (variants if isinstance(variants, list) else [variants]) + ], lambda v: v['transcripts'].keys() if 'transcripts' in v else {t['geneId'] for t in v['sortedTranscriptConsequences']}) + + +def _get_gene_aggs_for_cached_variants(variants, get_variant_genes): gene_aggs = defaultdict(lambda: {'total': 0, 'families': defaultdict(int)}) - # ES caches compound hets separately from main results, hail search caches everything together - flattened_variants = backend_specific_call( - lambda results: results, - lambda results: [v for variants in results for v in (variants if isinstance(variants, list) else [variants])], - lambda results: [v for variants in results for v in (variants if isinstance(variants, list) else [variants])], - )(previous_search_results['all_results']) - for var in flattened_variants: - # ES only reports breakdown for main transcript gene only, hail backend reports for all genes - gene_ids = backend_specific_call( - lambda variant: next(( - [gene_id] for gene_id, transcripts in variant['transcripts'].items() - if any(t['transcriptId'] == var['mainTranscriptId'] for t in transcripts) - ), []) if var['mainTranscriptId'] else [], - lambda variant: variant['transcripts'].keys(), - lambda variant: variant['transcripts'].keys() if 'transcripts' in variant else {t['geneId'] for t in variant['sortedTranscriptConsequences']}, - )(var) + for var in variants: + gene_ids = get_variant_genes(var) for gene_id in gene_ids: gene_aggs[gene_id]['total'] += 1 for family_guid in var['familyGuids']: @@ -464,12 +401,12 @@ def _parse_variant_items(search_json): def parse_variant_id(variant_id): try: - return parse_valid_variant_id(variant_id) + return _parse_valid_variant_id(variant_id) except (KeyError, ValueError): return None -def parse_valid_variant_id(variant_id): +def _parse_valid_variant_id(variant_id): chrom, pos, ref, alt = variant_id.split('-') chrom = format_chrom(chrom) pos = int(pos) @@ -482,17 +419,18 @@ def _validate_sort(sort, families): raise InvalidSearchException('Phenotype sort is only supported for single-family search.') -def _search_dataset_type(search): - locus = search['parsed_locus'] - parsed_variant_ids = locus.get('parsed_variant_ids', locus['variant_ids']) - rsids = locus.get('rs_ids') +def _search_dataset_type(search, genome_version): + parsed_variant_ids = search.get('parsed_variant_ids') + rsids = search.get('rs_ids') if parsed_variant_ids or rsids: lookup_dataset_type = Sample.DATASET_TYPE_VARIANT_CALLS if rsids else _variant_ids_dataset_type(parsed_variant_ids) return Sample.DATASET_TYPE_VARIANT_CALLS, None, lookup_dataset_type - intervals = (locus['intervals'] or (locus.get('gene_intervals') or {}).values()) if 'exclude_intervals' in locus and not locus['exclude_intervals'] else None - dataset_type = _annotation_dataset_type(search.get('annotations'), intervals, pathogenicity=search.get('pathogenicity')) - secondary_dataset_type = _annotation_dataset_type(search['annotations_secondary'], intervals) if search.get('annotations_secondary') else None + chroms = [gene[f'chromGrch{genome_version}'] for gene in (search.get('genes') or {}).values()] + [ + interval['chrom'] for interval in (search.get('intervals') or []) + ] if not search.get('exclude_locations') else None + dataset_type = _annotation_dataset_type(search.get('annotations'), chroms, pathogenicity=search.get('pathogenicity')) + secondary_dataset_type = _annotation_dataset_type(search['annotations_secondary'], chroms) if search.get('annotations_secondary') else None return dataset_type, secondary_dataset_type, None @@ -502,19 +440,19 @@ def _variant_ids_dataset_type(all_variant_ids): any_sv = len(variant_ids) < len(all_variant_ids) if len(variant_ids) == 0: return Sample.DATASET_TYPE_SV_CALLS - return _chromosome_filter_dataset_type(variant_ids, any_sv) + return _chromosome_filter_dataset_type([vid[0] for vid in variant_ids], any_sv) -def _chromosome_filter_dataset_type(loci, any_sv): - has_mito = [locus[0] for locus in loci if locus[0].replace('chr', '').startswith('M')] - if len(has_mito) == len(loci): +def _chromosome_filter_dataset_type(chroms, any_sv): + has_mito = [chrom for chrom in chroms if chrom.replace('chr', '').startswith('M')] + if len(has_mito) == len(chroms): return Sample.DATASET_TYPE_MITO_CALLS elif not has_mito: return DATASET_TYPE_NO_MITO if any_sv else DATASET_TYPE_SNP_INDEL_ONLY return ALL_DATA_TYPES if any_sv else Sample.DATASET_TYPE_VARIANT_CALLS -def _annotation_dataset_type(annotations, intervals, pathogenicity=None): - if not (annotations or intervals): +def _annotation_dataset_type(annotations, chroms, pathogenicity=None): + if not (annotations or chroms): return Sample.DATASET_TYPE_VARIANT_CALLS if pathogenicity else None annotation_types = set((annotations or {}).keys()) @@ -522,8 +460,8 @@ def _annotation_dataset_type(annotations, intervals, pathogenicity=None): return Sample.DATASET_TYPE_SV_CALLS no_svs = (annotations and annotation_types.isdisjoint(SV_ANNOTATION_TYPES)) - if intervals: - return _chromosome_filter_dataset_type(intervals, any_sv=not no_svs) + if chroms: + return _chromosome_filter_dataset_type(chroms, any_sv=not no_svs) elif no_svs: return Sample.DATASET_TYPE_VARIANT_CALLS return ALL_DATA_TYPES @@ -558,7 +496,7 @@ def _parse_inheritance(search, samples): def _validate_search(search, samples, previous_search_results): has_comp_het_search = search.get('inheritance_mode') in {RECESSIVE, COMPOUND_HET} and not previous_search_results.get('grouped_results') - has_location_filter = any(search['parsed_locus'].get(field) for field in ['genes', 'gene_ids', 'intervals', 'variant_ids']) + has_location_filter = any(search.get(field) for field in ['genes', 'intervals', 'variant_ids']) if has_comp_het_search: if not search.get('annotations'): raise InvalidSearchException('Annotations must be specified to search for compound heterozygous variants') @@ -579,7 +517,7 @@ def _validate_search(search, samples, previous_search_results): ) if not has_location_filter: - backend_specific_call(lambda *args: None, _validate_no_location_search, _validate_no_location_search)(samples) + backend_specific_call(lambda *args: None, _validate_no_location_search)(samples) MAX_FAMILY_COUNTS = {Sample.SAMPLE_TYPE_WES: 200, Sample.SAMPLE_TYPE_WGS: 35} @@ -598,7 +536,7 @@ def _validate_no_location_search(samples): def _filter_inheritance_family_samples(samples, inheritance_filter): family_groups = defaultdict(set) - sample_group_field = backend_specific_call('elasticsearch_index', 'dataset_type', 'dataset_type') + sample_group_field = backend_specific_call('elasticsearch_index', 'dataset_type') individual_affected_status = inheritance_filter.get('affected') or {} genotype_filter = None if inheritance_filter.get(Individual.AFFECTED_STATUS_AFFECTED) else inheritance_filter.get('genotype') for sample in samples: @@ -622,37 +560,6 @@ def _filter_inheritance_family_samples(samples, inheritance_filter): ] -def _parse_locus_gene_intervals(genome_version, genes=None, intervals=None, rs_ids=None, parsed_variant_ids=None, exclude_locations=False, **kwargs): - intervals = [_format_interval(**interval) for interval in intervals or []] - gene_intervals = gene_ids = None - if genes: - gene_intervals = { - gene['id']: [gene[f'{field}Grch{genome_version}'] for field in ['chrom', 'start', 'end']] for gene in genes.values() - } - if exclude_locations: - intervals += sorted(gene_intervals.values()) - gene_intervals = None - else: - gene_ids = sorted(genes.keys()) - - return { - 'intervals': intervals or None, - 'gene_intervals': gene_intervals, - 'exclude_intervals': exclude_locations, - 'gene_ids': gene_ids, - 'variant_ids': parsed_variant_ids, - 'rs_ids': rs_ids, - } - - -def _parse_locus_intervals(*args, **kwargs): - parsed_locus = _parse_locus_gene_intervals(*args, **kwargs) - gene_intervals = parsed_locus.pop('gene_intervals') - if gene_intervals: - parsed_locus['intervals'] = (parsed_locus['intervals'] or []) + sorted(gene_intervals.values()) - return parsed_locus - - def _format_interval(chrom=None, start=None, end=None, offset=None, **kwargs): if offset: offset_pos = int((end - start) * offset) diff --git a/seqr/utils/vcf_utils.py b/seqr/utils/vcf_utils.py index d9f49814a3..bf38cde5d1 100644 --- a/seqr/utils/vcf_utils.py +++ b/seqr/utils/vcf_utils.py @@ -82,25 +82,20 @@ def validate_vcf_and_get_samples(data_path, user, genome_version, path_name=None return None byte_range = None if vcf_filename.endswith('.vcf') else (0, BLOCK_SIZE) - samples = {} - header = [] meta = defaultdict(dict) - for line in file_iter(vcf_filename, byte_range=byte_range): - line_str = line.decode() if isinstance(line, bytes) else line - if line_str.startswith('#'): - if line_str.startswith('#CHROM'): - header_cols = line_str.rstrip().split('\t') - format_indices = [index for index, col in enumerate(header_cols) if col == 'FORMAT'] - format_index = format_indices[0] + 1 if format_indices else len(header_cols) - header = header_cols[0:format_index] - samples = set(header_cols[format_index:]) - break - else: - meta_info = _get_vcf_meta_info(line_str) - if meta_info: - meta[meta_info['field']].update({meta_info['id']: meta_info['type']}) - else: - raise ErrorsWarningsException(['No header found in the VCF file.'], []) + try: + header_line = next(_get_vcf_header_line(file_iter(vcf_filename, byte_range=byte_range), meta)) + except StopIteration: + raise ErrorsWarningsException(['No header found in the VCF file.'], []) + except UnicodeDecodeError: + raise ErrorsWarningsException([ + 'Unable to read the VCF file. This often occurs when a file is improperly gzipped, or when the file extension does not align with the file type (i.e. a .gz file that is not actually gzipped)' + ], []) + header_cols = header_line.rstrip().split('\t') + format_indices = [index for index, col in enumerate(header_cols) if col == 'FORMAT'] + format_index = format_indices[0] + 1 if format_indices else len(header_cols) + header = header_cols[0:format_index] + samples = set(header_cols[format_index:]) _validate_vcf_header(header) if not samples: @@ -110,6 +105,18 @@ def validate_vcf_and_get_samples(data_path, user, genome_version, path_name=None return samples +def _get_vcf_header_line(vcf_file, meta): + for line in vcf_file: + line_str = line.decode() if isinstance(line, bytes) else line + if line_str.startswith('#'): + if line_str.startswith('#CHROM'): + yield line_str + else: + meta_info = _get_vcf_meta_info(line_str) + if meta_info: + meta[meta_info['field']].update({meta_info['id']: meta_info['type']}) + + def _validate_vcf_exists(data_path, user, path_name, allowed_exts): file_extensions = (allowed_exts or ()) + VCF_FILE_EXTENSIONS if not data_path.endswith(file_extensions): diff --git a/seqr/views/apis/anvil_workspace_api_tests.py b/seqr/views/apis/anvil_workspace_api_tests.py index 91bbfd8ed6..f7e5a79cf6 100644 --- a/seqr/views/apis/anvil_workspace_api_tests.py +++ b/seqr/views/apis/anvil_workspace_api_tests.py @@ -369,6 +369,14 @@ def test_validate_anvil_vcf(self, mock_subprocess, mock_file_logger, mock_utils_ mock.call('==> gsutil cat -r 0-65536 gs://test_bucket/test_path.vcf.gz | gunzip -c -q - ', None), ]) + # test improperly encoded file + mock_subprocess.return_value.stdout = [b'\x80'] + response = self.client.post(url, content_type='application/json', data=json.dumps(REQUEST_BODY_GZ_DATA_PATH)) + self.assertEqual(response.status_code, 400) + self.assertListEqual(response.json()['errors'], [ + 'Unable to read the VCF file. This often occurs when a file is improperly gzipped, or when the file extension does not align with the file type (i.e. a .gz file that is not actually gzipped)', + ]) + # test header errors mock_subprocess.return_value.stdout = BASIC_META + BAD_INFO_META + BAD_FORMAT_META + BAD_HEADER_LINE + DATA_LINES response = self.client.post(url, content_type='application/json', data=json.dumps(REQUEST_BODY_GZ_DATA_PATH)) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 5910d97003..7dfde66a4e 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -16,7 +16,8 @@ from clickhouse_search.search import delete_clickhouse_project, delete_clickhouse_family from seqr.utils.communication_utils import send_project_notification from seqr.utils.search.add_data_utils import prepare_data_loading_request, get_loading_samples_validator -from seqr.utils.search.utils import get_search_backend_status, delete_search_backend_data, backend_specific_call +from seqr.utils.search.elasticsearch.es_utils import get_elasticsearch_status, delete_es_index +from seqr.utils.search.utils import clickhouse_only, es_only, InvalidSearchException from seqr.utils.file_utils import file_iter, does_file_exist from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException @@ -43,14 +44,21 @@ @data_manager_required +@es_only def elasticsearch_status(request): - return create_json_response(get_search_backend_status()) + return create_json_response(get_elasticsearch_status()) @data_manager_required +@es_only def delete_index(request): index = json.loads(request.body)['index'] - updated_indices = delete_search_backend_data(index) + active_samples = Sample.objects.filter(is_active=True, elasticsearch_index=index) + if active_samples: + projects = set(active_samples.values_list('individual__family__project__name', flat=True)) + raise InvalidSearchException(f'"{index}" is still used by: {", ".join(projects)}') + + updated_indices = delete_es_index(index) return create_json_response({'indices': updated_indices}) @@ -424,6 +432,7 @@ def fetch_missing_vcf_samples(missing_vcf_samples): @data_manager_required +@clickhouse_only def trigger_delete_project(request): request_json = json.loads(request.body) project_guid = request_json.pop('project') @@ -432,6 +441,7 @@ def trigger_delete_project(request): @data_manager_required +@clickhouse_only def trigger_delete_family(request): request_json = json.loads(request.body) family_guid = request_json.pop('family') @@ -439,10 +449,6 @@ def trigger_delete_family(request): return _trigger_data_update(delete_clickhouse_family, request_json, project, family_guid) -def _raise_backend_not_implemented(*args, **kwargs): - raise ErrorsWarningsException(['This functionality is not available in the current search backend']) - - def _trigger_data_update(clickhouse_func, request_json, project, *args): dataset_type = request_json.get('datasetType') sample_types = Sample.objects.filter( @@ -450,9 +456,7 @@ def _trigger_data_update(clickhouse_func, request_json, project, *args): ).values_list('sample_type', flat=True).distinct() if dataset_type == Sample.DATASET_TYPE_SV_CALLS else [None] info = [] for sample_type in sample_types: - info.append(backend_specific_call( - _raise_backend_not_implemented, _raise_backend_not_implemented, clickhouse_func, - )(project, *args, dataset_type=dataset_type, sample_type=sample_type)) + info.append(clickhouse_func(project, *args, dataset_type=dataset_type, sample_type=sample_type)) return create_json_response({'info': info}) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index ed5ce585b0..035be59d6d 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -226,8 +226,8 @@ } EXPECTED_ERRORS = [ + 'test_index_mito_wgs does not exist and is used by project(s) 1kg project n\xe5me with uni\xe7\xf8de (1 samples)', 'test_index_old does not exist and is used by project(s) 1kg project n\xe5me with uni\xe7\xf8de (1 samples)', - 'test_index_mito_wgs does not exist and is used by project(s) 1kg project n\xe5me with uni\xe7\xf8de (1 samples)' ] RNA_TPM_MUSCLE_SAMPLE_GUID = 'RS000162_T_na19675_d2' @@ -506,10 +506,7 @@ def test_delete_index(self): self.check_data_manager_login(url) response = self.client.post(url, content_type='application/json', data=json.dumps({'index': 'test_index'})) - self.assertEqual(response.status_code, 400) - self.assertEqual( - response.json()['error'], '"test_index" is still used by: 1kg project n\xe5me with uni\xe7\xf8de') - self.assertEqual(len(urllib3_responses.calls), 0) + self._assert_expected_delete_active_index_response(response) urllib3_responses.add_json( '/_cat/indices?format=json&h=index,docs.count,store.size,creation.date.string', ES_CAT_INDICES) @@ -520,6 +517,12 @@ def test_delete_index(self): response = self.client.post(url, content_type='application/json', data=json.dumps({'index': 'unused_index'})) self._assert_expected_delete_index_response(response) + def _assert_expected_delete_active_index_response(self, response): + self.assertEqual(response.status_code, 400) + self.assertEqual( + response.json()['error'], '"test_index" is still used by: 1kg project n\xe5me with uni\xe7\xf8de') + self.assertEqual(len(urllib3_responses.calls), 0) + def _assert_expected_delete_index_response(self, response): self.assertEqual(response.status_code, 200) response_json = response.json() @@ -1567,14 +1570,12 @@ def test_trigger_delete_family(self): self._assert_expected_delete_family(response) def _assert_expected_delete_project(self, response): - self._assert_expected_search_data_update(response) + self.assertEqual(response.status_code, 500) + self.assertDictEqual(response.json(), {'error': 'trigger_delete_project is disabled without the clickhouse backend'}) def _assert_expected_delete_family(self, response): - self._assert_expected_search_data_update(response) - - def _assert_expected_search_data_update(self, response): - self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'errors': ['This functionality is not available in the current search backend'], 'warnings': None}) + self.assertEqual(response.status_code, 500) + self.assertDictEqual(response.json(), {'error': 'trigger_delete_family is disabled without the clickhouse backend'}) class LocalDataManagerAPITest(AuthenticationTestCase, DataManagerAPITest): @@ -1806,12 +1807,15 @@ def _additional_expected_loading_subprocess_calls(file_path): return [f'gsutil mv tmp/temp_uploads/{file_path} gs://seqr-scratch-temp/{file_path}'] def _assert_expected_es_status(self, response): - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['error'], 'Elasticsearch is disabled') + self.assertEqual(response.status_code, 500) + self.assertEqual(response.json()['error'], 'elasticsearch_status is disabled without the elasticsearch backend') def _assert_expected_delete_index_response(self, response): - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['error'], 'Deleting indices is disabled without the elasticsearch backend') + self.assertEqual(response.status_code, 500) + self.assertEqual(response.json()['error'], 'delete_index is disabled without the elasticsearch backend') + + def _assert_expected_delete_active_index_response(self, response): + self._assert_expected_delete_index_response(response) def _assert_expected_get_projects_requests(self): pdo_filter = "OR(SEARCH('Methods (Loading)',ARRAYJOIN(PDOStatus,';')),SEARCH('On hold for phenotips, but ready to load',ARRAYJOIN(PDOStatus,';')))" @@ -2081,18 +2085,3 @@ def _assert_expected_airtable_errors(self, url): self.assertDictEqual(response.json(), { 'error': 'The following samples are associated with misconfigured PDOs in Airtable: HG00731, NA21234', }) - -@mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') -class HailBackendDataManagerAPITest(LocalDataManagerAPITest): - fixtures = ['users', 'social_auth', '1kg_project', 'reference_data'] - - CLICKHOUSE_HOSTNAME = '' - ES_HOSTNAME = '' - - def _assert_expected_es_status(self, response): - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['error'], 'Elasticsearch is disabled') - - def _assert_expected_delete_index_response(self, response): - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['error'], 'Deleting indices is disabled without the elasticsearch backend') diff --git a/seqr/views/apis/dataset_api.py b/seqr/views/apis/dataset_api.py index 078ccb0dda..e0dfa35a1d 100644 --- a/seqr/views/apis/dataset_api.py +++ b/seqr/views/apis/dataset_api.py @@ -1,13 +1,17 @@ import json -from seqr.utils.search.add_data_utils import add_new_es_search_samples -from seqr.models import Individual, Family +from seqr.utils.search.add_data_utils import basic_notify_search_data_loaded +from seqr.utils.search.elasticsearch.es_utils import validate_es_index_metadata_and_get_samples +from seqr.utils.search.utils import es_only +from seqr.models import Individual, Family, Sample +from seqr.views.utils.dataset_utils import match_and_update_search_samples, load_mapping_file from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.orm_to_json_utils import get_json_for_samples from seqr.views.utils.permissions_utils import get_project_and_check_permissions, data_manager_required @data_manager_required +@es_only def add_variants_dataset_handler(request, project_guid): """Create or update samples for the given variant dataset @@ -31,7 +35,7 @@ def add_variants_dataset_handler(request, project_guid): request_json = json.loads(request.body) try: - inactivated_sample_guids, updated_family_guids, updated_samples = add_new_es_search_samples( + inactivated_sample_guids, updated_family_guids, updated_samples = _add_new_es_search_samples( request_json, project, request.user, notify=True) except ValueError as e: return create_json_response({'errors': [str(e)]}, status=400) @@ -43,6 +47,37 @@ def add_variants_dataset_handler(request, project_guid): return create_json_response(response_json) +def _add_new_es_search_samples(request_json, project, user, notify=False, expected_families=None): + dataset_type = request_json.get('datasetType') + if dataset_type not in Sample.DATASET_TYPE_LOOKUP: + raise ValueError(f'Invalid dataset type "{dataset_type}"') + + sample_ids, sample_type, sample_data = validate_es_index_metadata_and_get_samples(request_json, project) + if not sample_ids: + raise ValueError('No samples found. Make sure the specified caller type is correct') + + sample_id_to_individual_id_mapping = load_mapping_file( + request_json['mappingFilePath'], user) if request_json.get('mappingFilePath') else {} + ignore_extra_samples = request_json.get('ignoreExtraSamplesInCallset') + sample_project_tuples = [(sample_id, project.name) for sample_id in sample_ids] + new_samples, updated_samples, inactivated_sample_guids, updated_family_guids = match_and_update_search_samples( + projects=[project], + user=user, + sample_project_tuples=sample_project_tuples, + sample_data=sample_data, + sample_type=sample_type, + dataset_type=dataset_type, + expected_families=expected_families, + sample_id_to_individual_id_mapping=sample_id_to_individual_id_mapping, + raise_unmatched_error_template=None if ignore_extra_samples else 'Matches not found for sample ids: {sample_ids}. Uploading a mapping file for these samples, or select the "Ignore extra samples in callset" checkbox to ignore.' + ) + + if notify: + basic_notify_search_data_loaded(project, dataset_type, sample_type, new_samples.values_list('sample_id')) + + return inactivated_sample_guids, updated_family_guids, updated_samples + + def _get_samples_json(updated_samples, inactivated_sample_guids, project_guid): updated_sample_json = get_json_for_samples(updated_samples, project_guid=project_guid) sample_response = {sample_guid: {'isActive': False} for sample_guid in inactivated_sample_guids} diff --git a/seqr/views/apis/dataset_api_tests.py b/seqr/views/apis/dataset_api_tests.py index 1428c80f4d..091f3abc08 100644 --- a/seqr/views/apis/dataset_api_tests.py +++ b/seqr/views/apis/dataset_api_tests.py @@ -284,6 +284,9 @@ def test_add_variants_dataset_errors(self): url = reverse(add_variants_dataset_handler, args=[PROJECT_GUID]) self.check_data_manager_login(url) + self._assert_expected_add_dataset_errors(url) + + def _assert_expected_add_dataset_errors(self, url): response = self.client.post(url, content_type='application/json', data=json.dumps({})) self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'errors': ['Invalid dataset type "None"']}) @@ -293,9 +296,6 @@ def test_add_variants_dataset_errors(self): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'errors': ['Invalid dataset type "NOT_A_TYPE"']}) - self._assert_expected_add_dataset_errors(url) - - def _assert_expected_add_dataset_errors(self, url): response = self.client.post(url, content_type='application/json', data=json.dumps({'datasetType': 'SV'})) self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'errors': ['request must contain field: "elasticsearchIndex"']}) @@ -463,8 +463,8 @@ class AnvilDatasetAPITest(AnvilAuthenticationTestCase, DatasetAPITest): def _assert_expected_add_dataset_errors(self, url): response = self.client.post(url, content_type='application/json', data=ADD_DATASET_PAYLOAD) - self.assertEqual(response.status_code, 400) - self.assertEqual(response.json()['errors'][0], 'Adding samples is disabled without the elasticsearch backend') + self.assertEqual(response.status_code, 500) + self.assertEqual(response.json()['error'], 'add_variants_dataset_handler is disabled without the elasticsearch backend') def test_add_variants_dataset(self, *args): # Adding dataset is always disabled when ES is disabled, which is tested in test_add_variants_dataset_errors diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index af71b1b34c..e26037621c 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -12,6 +12,7 @@ from matchmaker.models import MatchmakerSubmission from reference_data.models import Omim from seqr.utils.gene_utils import get_genes_for_variant_display +from seqr.utils.logging_utils import SeqrLogger from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.file_utils import save_uploaded_file, load_uploaded_file from seqr.views.utils.individual_utils import delete_individuals @@ -29,6 +30,8 @@ from seqr.views.utils.variant_utils import get_phenotype_prioritization, get_omim_intervals_query, DISCOVERY_CATEGORY from seqr.utils.xpos_utils import get_chrom_pos +logger = SeqrLogger(__name__) + FAMILY_ID_FIELD = 'familyId' PREVIOUS_FAMILY_ID_FIELD = 'previousFamilyId' @@ -53,14 +56,14 @@ def family_page_data(request, family_guid): add_families_context(response, families, project.guid, request.user, is_analyst, has_case_review_perm) family_response = response['familiesByGuid'][family_guid] - additional_fields = backend_specific_call([],[],['key', 'dataset_type']) + additional_fields = backend_specific_call([],['key', 'dataset_type']) discovery_variants = family.savedvariant_set.filter(varianttag__variant_tag_type__category=DISCOVERY_CATEGORY).values( 'xpos', 'xpos_end', *additional_fields, svType=F('saved_variant_json__svType'), transcripts=F('saved_variant_json__transcripts'), ) gene_ids = backend_specific_call( - _variants_gene_ids, _variants_gene_ids, _clickhouse_variants_gene_ids, - )(discovery_variants, project.genome_version) + _variants_gene_ids, _clickhouse_variants_gene_ids, + )(discovery_variants, project.genome_version, request.user) discovery_variant_intervals = [dict(zip( ['chrom', 'start', 'end_chrom', 'end', 'svType', 'hasSvType'], [*get_chrom_pos(v['xpos']), *get_chrom_pos(v['xpos_end']), v['svType'], (v.get('dataset_type') or '').startswith(Sample.DATASET_TYPE_SV_CALLS)] @@ -107,7 +110,7 @@ def _variants_gene_ids(variants, *args, **kwargs): return {gene_id for variant in variants for gene_id in (variant['transcripts'] or {}).keys()} -def _clickhouse_variants_gene_ids(variants, genome_version): +def _clickhouse_variants_gene_ids(variants, genome_version, user): keys_by_dataset_type = defaultdict(set) no_key_variants = [] for v in variants: @@ -117,7 +120,10 @@ def _clickhouse_variants_gene_ids(variants, genome_version): no_key_variants.append(v) gene_ids = _variants_gene_ids(no_key_variants) for dataset_type, keys in keys_by_dataset_type.items(): - gene_ids.update(get_clickhouse_genes(genome_version, dataset_type, keys)) + try: + gene_ids.update(get_clickhouse_genes(genome_version, dataset_type, keys)) + except Exception as e: + logger.error(f'Error loading genes from clickhouse: {e}', user) return gene_ids diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index 17a1dd9a7e..19c9b98f24 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -4,6 +4,7 @@ from copy import deepcopy from datetime import datetime from django.core.files.uploadedfile import SimpleUploadedFile +from django.db import connections from django.urls.base import reverse from matchmaker.models import MatchmakerSubmission @@ -183,6 +184,39 @@ def test_family_page_data(self): response = self.client.get(url.replace(FAMILY_GUID, 'invalid_guid')) self.assertEqual(response.status_code, 404) + # Test family with clickhouse saved variants + omim_options = { + '615120': {'phenotypeMimNumber': 615120, 'phenotypes': [{ + 'geneSymbol': 'RP11', 'mimNumber': 103320, 'phenotypeMimNumber': 615120, + 'phenotypeDescription': 'Myasthenic syndrome, congenital, 8, with pre- and postsynaptic defects', + 'phenotypeInheritance': 'Autosomal recessive, X-linked recessive', 'chrom': '1', 'start': 29554, + 'end': 31109, + }]}, + '616126': {'phenotypeMimNumber': 616126, 'phenotypes': [{ + 'geneSymbol': 'OR4G11P', 'mimNumber': 147571, 'phenotypeMimNumber': 616126, + 'phenotypeDescription': 'Immunodeficiency 38', + 'phenotypeInheritance': 'Autosomal recessive', 'chrom': '1', 'start': 11869, + 'end': 14409, + }]}, + } + self._assert_expected_family2_data(omim_options, url.replace(FAMILY_GUID, FAMILY_GUID2), response_keys, family_fields) + + def _assert_expected_family2_data(self, omim_options, family_2_url, response_keys, family_fields): + response = self.client.get(family_2_url) + self.assertEqual(response.status_code, 200) + response_json = response.json() + self.assertSetEqual(set(response_json.keys()), response_keys) + family = response_json['familiesByGuid'][FAMILY_GUID2] + self.assertSetEqual(set(family.keys()), family_fields) + self.assertDictEqual(family['postDiscoveryOmimOptions'], omim_options) + self.assertEqual(family['projectGuid'], PROJECT_GUID) + self.assertSetEqual(set(family['individualGuids']), set(response_json['individualsByGuid'].keys())) + self.assertEqual(len(response_json['individualsByGuid']), 3) + self.assertEqual(len(response_json['samplesByGuid']), 7) + self.assertEqual(len(response_json['igvSamplesByGuid']), 0) + self.assertEqual(len(response_json['mmeSubmissionsByGuid']), 0) + self.assertEqual(len(response_json['familyNotesByGuid']), 0) + def test_family_variant_tag_summary(self): url = reverse(family_variant_tag_summary, args=[FAMILY_GUID]) self.check_collaborator_login(url) @@ -198,7 +232,11 @@ def test_family_variant_tag_summary(self): family = response_json['familiesByGuid'][FAMILY_GUID] self.assertSetEqual(set(family.keys()), {'familyGuid', 'discoveryTags'}) - self.assertSetEqual({tag['variantGuid'] for tag in family['discoveryTags']}, {'SV0000001_2103343353_r0390_100'}) + self.assertListEqual(family['discoveryTags'], [{ + 'transcripts': {'ENSG00000135953': [mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY]}, + 'mainTranscriptId': 'ENST00000258436', + 'selectedMainTranscriptId': None, + }]) project = response_json['projectsByGuid'][PROJECT_GUID] self.assertSetEqual(set(project.keys()), {'variantTagTypes', 'variantFunctionalTagTypes'}) @@ -669,3 +707,17 @@ class AnvilFamilyAPITest(AnvilAuthenticationTestCase, FamilyAPITest): fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_saved_variants'] EXTERNAL_ANVIL_CAN_DELETE = True + + def _assert_expected_family2_data(self, omim_options, *args, **kwargs): + super()._assert_expected_family2_data(omim_options, *args, **kwargs) + + # test graceful handling when clickhouse is down + self.reset_logs() + connections['clickhouse'].close() + super()._assert_expected_family2_data({}, *args, **kwargs) + self.assert_json_logs(self.analyst_user, [ + ("Error loading genes from clickhouse: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.", { + 'severity': 'ERROR', + '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', + }), + ]) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index 7aee432b5f..b896e2368e 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -3,6 +3,7 @@ from copy import deepcopy from datetime import datetime from django.contrib.auth.models import Group +from django.db import connections from django.urls.base import reverse import responses @@ -40,11 +41,20 @@ {'id': 'recSgwrXNkmlIB5eM', 'fields': {'Status': 'Available in Seqr'}}, ]} +DISCOVERY_TAG = { + 'mainTranscriptId': None, + 'selectedMainTranscriptId': 'ENST00000371839', + 'transcripts': { + 'ENSG00000135953': mock.ANY, + 'ENSG00000240361': mock.ANY, + }, +} class ProjectAPITest(object): CREATE_PROJECT_JSON = WORKSPACE_CREATE_PROJECT_JSON REQUIRED_FIELDS = ['name', 'genomeVersion', 'workspaceNamespace', 'workspaceName'] AIRTABLE_TRACKING_URL = f'{MOCK_AIRTABLE_URL}/appUelDNM3BnWaR7M/AnVIL%20Seqr%20Loading%20Requests%20Tracking' + DISCOVERY_TAG = DISCOVERY_TAG @mock.patch('seqr.views.utils.airtable_utils.logger') @mock.patch('seqr.views.utils.airtable_utils.AIRTABLE_URL', MOCK_AIRTABLE_URL) @@ -372,11 +382,19 @@ def test_project_families(self): url = reverse(project_families, args=[PROJECT_GUID]) self.check_collaborator_login(url) + response_keys = {'familiesByGuid', 'genesById'} + + empty_url = reverse(project_families, args=[EMPTY_PROJECT_GUID]) + self._check_empty_project(empty_url, response_keys) + + gene_ids = self._assert_expected_project_families(url, response_keys) + self.assertSetEqual(gene_ids, {'ENSG00000135953', 'ENSG00000240361'}) + + def _assert_expected_project_families(self, url, response_keys): response = self.client.get(url) self.assertEqual(response.status_code, 200) response_json = response.json() - response_keys = {'familiesByGuid', 'genesById'} self.assertSetEqual(set(response_json.keys()), response_keys) family_1 = response_json['familiesByGuid']['F000001_1'] @@ -418,20 +436,18 @@ def test_project_families(self): self.assertListEqual(family_3['discoveryTags'], []) self.assertListEqual(empty_family['discoveryTags'], []) - self.assertSetEqual({tag['variantGuid'] for tag in family_1['discoveryTags']}, {'SV0000001_2103343353_r0390_100'}) - self.assertSetEqual( - {tag['variantGuid'] for tag in response_json['familiesByGuid']['F000002_2']['discoveryTags']}, - {'SV0000002_1248367227_r0390_100'}) + self.assertListEqual(family_1['discoveryTags'], [{ + 'transcripts': {'ENSG00000135953': [mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY]}, + 'mainTranscriptId': 'ENST00000258436', + 'selectedMainTranscriptId': None, + }]) + self.assertListEqual(response_json['familiesByGuid']['F000002_2']['discoveryTags'], [self.DISCOVERY_TAG]) no_discovery_families = set(response_json['familiesByGuid'].keys()) - {'F000001_1', 'F000002_2'} self.assertSetEqual({ len(response_json['familiesByGuid'][family_guid]['discoveryTags']) for family_guid in no_discovery_families }, {0}) - self.assertSetEqual(set(response_json['genesById'].keys()), {'ENSG00000135953', 'ENSG00000240361'}) - - # Test empty project - empty_url = reverse(project_families, args=[EMPTY_PROJECT_GUID]) - self._check_empty_project(empty_url, response_keys) + return set(response_json['genesById'].keys()) def test_project_individuals(self): url = reverse(project_individuals, args=[PROJECT_GUID]) @@ -704,6 +720,7 @@ class AnvilProjectAPITest(AnvilAuthenticationTestCase, ProjectAPITest): PROJECT_COLLABORATORS = ANVIL_COLLABORATORS PROJECT_COLLABORATOR_GROUPS = None HAS_EMPTY_PROJECT = False + DISCOVERY_TAG = {**DISCOVERY_TAG, 'mainTranscriptId': 'ENST00000505820'} def test_create_and_delete_project(self, *args, **kwargs): super(AnvilProjectAPITest, self).test_create_and_delete_project(*args, **kwargs) @@ -777,3 +794,22 @@ def test_project_collaborators(self): 'my-seqr-billing', 'anvil-1kg project n\u00e5me with uni\u00e7\u00f8de') self.assertEqual(self.mock_get_ws_access_level.call_count, 5) + + def _assert_expected_project_families(self, *args, **kwargs): + gene_ids = super()._assert_expected_project_families(*args, **kwargs) + + # Test success when clickhouse is unavailable + self.reset_logs() + connections['clickhouse'].close() + self.DISCOVERY_TAG = {**DISCOVERY_TAG, 'transcripts': {}} + no_clickhouse_gene_ids = super()._assert_expected_project_families(*args, **kwargs) + self.assertSetEqual(no_clickhouse_gene_ids, {'ENSG00000135953'}) + self.assert_json_logs(None, [ + ("Error loading discovery genes from clickhouse: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.", { + 'severity': 'ERROR', + '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', + }), + ]) + + return gene_ids + diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 0044ea276b..8a75fb697b 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -8,7 +8,8 @@ from clickhouse_search.search import get_annotations_queryset, get_transcripts_queryset from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\ Family, GeneNote, Project -from seqr.utils.search.utils import backend_specific_call +from seqr.utils.search.elasticsearch.es_utils import update_project_saved_variant_json +from seqr.utils.search.utils import backend_specific_call, es_only from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json from seqr.views.utils.json_utils import create_json_response @@ -16,8 +17,7 @@ get_json_for_saved_variants_child_entities, get_json_for_gene_notes_by_gene_id, STRUCTURED_METADATA_TAG_TYPES from seqr.views.utils.permissions_utils import get_project_and_check_permissions, check_project_permissions, \ login_and_policies_required -from seqr.views.utils.variant_utils import update_project_saved_variant_json, reset_cached_search_results, \ - get_variants_response, parse_saved_variant_json +from seqr.views.utils.variant_utils import reset_cached_search_results, get_variants_response, parse_saved_variant_json logger = logging.getLogger(__name__) @@ -127,7 +127,7 @@ def _create_variant_note(saved_variants, note_json, user, genome_version): response = {} if note_json.get('saveAsGeneNote'): - gene_id = backend_specific_call(_variant_gene_id, _variant_gene_id, _clickhouse_variant_gene_id)(saved_variants[0], genome_version) + gene_id = backend_specific_call(_variant_gene_id, _clickhouse_variant_gene_id)(saved_variants[0], genome_version) create_model_from_json(GeneNote, {'note': note_json.get('note'), 'gene_id': gene_id}, user) response['genesById'] = {gene_id: { 'notes': get_json_for_gene_notes_by_gene_id([gene_id], user)[gene_id], @@ -319,8 +319,8 @@ def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=Vari @login_and_policies_required +@es_only def update_saved_variant_json(request, project_guid): - backend_specific_call(lambda: True, _backend_error, _backend_error)() project = get_project_and_check_permissions(project_guid, request.user, can_edit=True) reset_cached_search_results(project) try: @@ -332,10 +332,6 @@ def update_saved_variant_json(request, project_guid): return create_json_response({variant_guid: None for variant_guid in updated_saved_variant_guids or []}) -def _backend_error(*args, **kwargs): - raise ValueError('Endpoint is disabled for current search backend') - - @login_and_policies_required def update_variant_main_transcript(request, variant_guid, transcript_id): saved_variant = SavedVariant.objects.get(guid=variant_guid) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 98b235f636..9c13be3624 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -4,7 +4,7 @@ from django.urls.base import reverse -from seqr.models import SavedVariant, VariantNote, VariantTag, VariantFunctionalData, Family, Project +from seqr.models import SavedVariant, VariantNote, VariantTag, VariantFunctionalData, Sample, Project from seqr.views.apis.saved_variant_api import saved_variant_data, create_variant_note_handler, create_saved_variant_handler, \ update_variant_note_handler, delete_variant_note_handler, update_variant_tags_handler, update_saved_variant_json, \ update_variant_main_transcript, update_variant_functional_data_handler, update_variant_acmg_classification_handler @@ -953,20 +953,19 @@ def test_update_compound_hets_variant_functional_data(self): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': 'Unable to find the following variant(s): not_variant'}) - @mock.patch('seqr.views.utils.variant_utils.MAX_VARIANTS_FETCH', 2) - @mock.patch('seqr.utils.search.utils.es_backend_enabled') + @mock.patch('seqr.utils.search.elasticsearch.es_utils.MAX_VARIANTS_FETCH', 2) @mock.patch('seqr.views.apis.saved_variant_api.logger') - @mock.patch('seqr.views.utils.variant_utils.get_variants_for_variant_ids') - def test_update_saved_variant_json(self, mock_get_variants, mock_logger, mock_es_enabled): - mock_es_enabled.return_value = True - mock_get_variants.side_effect = lambda families, variant_ids, **kwargs: \ - [{'variantId': variant_id, 'familyGuids': [family.guid for family in families]} + @mock.patch('seqr.utils.search.elasticsearch.es_utils.get_es_variants_for_variant_ids') + def test_update_saved_variant_json(self, mock_get_variants, mock_logger): + mock_get_variants.side_effect = lambda samples, genome_version, variant_ids, user: \ + [{'variantId': variant_id, 'familyGuids': sorted({sample.individual.family.guid for sample in samples})} for variant_id in variant_ids] url = reverse(update_saved_variant_json, args=['R0001_1kg']) self.check_manager_login(url) response = self.client.post(url) + self.assertEqual(response.status_code, 200) self.assertDictEqual( @@ -975,11 +974,13 @@ def test_update_saved_variant_json(self, mock_get_variants, mock_logger, mock_es 'SV0059956_11560662_f019313_1': None} ) - families = [Family.objects.get(guid='F000001_1'), Family.objects.get(guid='F000002_2')] mock_get_variants.assert_has_calls([ - mock.call(families, ['1-248367227-TC-T', '1-46859832-G-A'], user=self.manager_user, user_email=None), - mock.call(families, ['21-3343353-GAGA-G'], user=self.manager_user, user_email=None), + mock.call(mock.ANY, '37', ['1-248367227-TC-T', '1-46859832-G-A'], self.manager_user), + mock.call(mock.ANY, '37', ['21-3343353-GAGA-G'], self.manager_user), ]) + samples = set(Sample.objects.filter(individual__family_id__in=[1, 2], is_active=True).values_list('id', flat=True)) + self.assertSetEqual(set(mock_get_variants.call_args_list[0].args[0].values_list('id', flat=True)), samples) + self.assertSetEqual(set(mock_get_variants.call_args_list[1].args[0].values_list('id', flat=True)), samples) mock_logger.error.assert_not_called() # Test handles update error @@ -988,10 +989,6 @@ def test_update_saved_variant_json(self, mock_get_variants, mock_logger, mock_es self.assertEqual(response.status_code, 200) mock_logger.error.assert_called_with('Unable to reset saved variant json for R0001_1kg: Unable to fetch variants') - mock_es_enabled.return_value = False - response = self.client.post(url) - self.assertEqual(response.status_code, 500) - def test_update_variant_main_transcript(self): transcript_id = 'ENST00000438943' update_main_transcript_url = reverse(update_variant_main_transcript, args=[VARIANT_GUID, transcript_id]) @@ -1106,8 +1103,11 @@ def test_update_compound_hets_variant_functional_data(self): assert_no_list_ws_has_al(self, 3) def test_update_saved_variant_json(self, *args): - super(AnvilSavedVariantAPITest, self).test_update_saved_variant_json(*args) - assert_no_list_ws_has_al(self, 3) + url = reverse(update_saved_variant_json, args=['R0001_1kg']) + self.check_require_login(url) + response = self.client.post(url) + self.assertEqual(response.status_code, 500) + self.assertDictEqual(response.json(), {'error': 'update_saved_variant_json is disabled without the elasticsearch backend'}) def test_update_variant_main_transcript(self): super(AnvilSavedVariantAPITest, self).test_update_variant_main_transcript() diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 7a9e9fdf70..1b0f7f621b 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -116,7 +116,6 @@ def saved_variants_page(request, tag): if gene: gene_filter = Q(saved_variant_json__transcripts__has_key=gene) | backend_specific_call( - lambda *args: Q(), lambda *args: Q(), _saved_variant_with_clickhouse_gene_q, )(saved_variant_models, gene, is_all_tags) diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index a917d78829..7df3cad33e 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -839,7 +839,7 @@ def _test_metadata_airtable_responses(self, include_airtable_url, expected_indiv self.assertEqual(response.status_code, 200) self._has_expected_metadata_response(response, expected_individuals) - @mock.patch('seqr.views.utils.variant_utils.get_variants_for_variant_ids', lambda *args, **kwargs: PARSED_VARIANTS) + @mock.patch('seqr.views.utils.variant_utils.get_es_variants_for_variant_ids', lambda *args, **kwargs: PARSED_VARIANTS) def test_bulk_update_family_external_analysis(self, *args, **kwargs): super().test_bulk_update_family_external_analysis(*args, **kwargs) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index c8d6781bce..4673f92bbf 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -15,7 +15,7 @@ from reference_data.models import GENOME_VERSION_GRCh37, GENOME_VERSION_GRCh38 from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, get_search_samples, \ - variant_lookup, sv_variant_lookup, parse_variant_id + variant_lookup, parse_variant_id from seqr.utils.search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY from seqr.utils.search.utils import InvalidSearchException from seqr.utils.xpos_utils import get_xpos @@ -544,32 +544,22 @@ def _flatten_variants(variants): flattened_variants.append(variant) return flattened_variants -def _parse_lookup_request(request): - kwargs = {_to_snake_case(k): v for k, v in request.GET.items()} - - variant_id = kwargs.pop('variant_id') - parsed_variant_id = parse_variant_id(variant_id) - return parsed_variant_id, variant_id, kwargs @login_and_policies_required def variant_lookup_handler(request): - parsed_variant_id, variant_id, kwargs = _parse_lookup_request(request) - is_sv = not parsed_variant_id - genome_version = kwargs.get('genome_version', GENOME_VERSION_GRCh38) - if is_sv: - families = _all_genome_version_families( - genome_version, request.user, - ) - if not families: - raise PermissionDenied() - variants = sv_variant_lookup(request.user, variant_id, families, **kwargs) - else: - variant = variant_lookup(request.user, parsed_variant_id, **kwargs) - variants = [variant] - families = Family.objects.filter( - guid__in=variant['familyGenotypes'], - project__guid__in=get_project_guids_user_can_view(request.user, limit_data_manager=True), - ) + variant_id = request.GET.get('variantId') + genome_version = request.GET.get('genomeVersion') or GENOME_VERSION_GRCh38 + variants = variant_lookup(request.user, variant_id, genome_version, sample_type=request.GET.get('sampleType')) + + family_guids = set() + for variant in variants: + family_guids.update(variant['familyGenotypes'].keys()) + + families = Family.objects.filter( + guid__in=family_guids, + project__guid__in=get_project_guids_user_can_view(request.user, limit_data_manager=True), + ) + for variant in variants: variant['familyGuids'] = list(families.values_list('guid', flat=True)) saved_variants, _ = _get_saved_variant_models(variants, None) if families else (None, None) @@ -579,17 +569,16 @@ def variant_lookup_handler(request): ) response['variants'] = variants - if not is_sv: - _update_lookup_variant(variant, response) - - return create_json_response(response) - - -def _update_lookup_variant(variant, response): individual_guid_map = { (i['familyGuid'], i['individualId']): i['individualGuid'] for i in response['individualsByGuid'].values() } + for variant in variants: + _update_lookup_variant(variant, response, individual_guid_map) + return create_json_response(response) + + +def _update_lookup_variant(variant, response, individual_guid_map): no_access_families = set(variant['familyGenotypes']) - set(variant['familyGuids']) individual_summary_map = { (i.pop('family__guid'), i.pop('individual_id')): (i.pop('guid'), i) @@ -601,11 +590,12 @@ def _update_lookup_variant(variant, response): add_individual_hpo_details([i for _, i in individual_summary_map.values()]) variant['genotypes'] = {} - variant['lookupFamilyGuids'] = sorted(variant.pop('familyGuids')) + variant['lookupFamilyGuids'] = sorted([guid for guid in variant.pop('familyGuids') if guid in variant['familyGenotypes']]) variant['familyGuids'] = [] for family_guid in variant['lookupFamilyGuids']: for genotype in variant['familyGenotypes'].pop(family_guid): individual_guid = individual_guid_map[(family_guid, genotype['sampleId'])] + genotype = {**genotype, 'individualGuid': individual_guid} if individual_guid in variant['genotypes']: genotype = [variant['genotypes'][individual_guid], genotype] variant['genotypes'][individual_guid] = genotype @@ -641,7 +631,9 @@ def _update_lookup_variant(variant, response): @login_and_policies_required def vlm_lookup_handler(request): - parsed_variant_id, _, kwargs = _parse_lookup_request(request) + kwargs = {_to_snake_case(k): v for k, v in request.GET.items()} + variant_id = kwargs.pop('variant_id') + parsed_variant_id = parse_variant_id(variant_id) if parsed_variant_id: invalid_alleles = [f'"{allele}"' for allele in parsed_variant_id[2:] if not re.fullmatch(r'[ATCG]+', allele)] if invalid_alleles: diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 2d3688a009..e637e90bad 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -7,7 +7,8 @@ from django.urls.base import reverse from elasticsearch.exceptions import ConnectionTimeout, TransportError -from clickhouse_search.test_utils import VARIANT2, VARIANT3, VARIANT_LOOKUP_VARIANT, GCNV_VARIANT1, SV_VARIANT1 +from clickhouse_search.test_utils import VARIANT2, VARIANT3, VARIANT_LOOKUP_VARIANT, GCNV_LOOKUP_VARIANT, \ + SV_LOOKUP_VARIANT, SV_VARIANT4, GCNV_VARIANT4 from seqr.models import VariantSearchResults, LocusList, Project, VariantSearch from seqr.utils.search.utils import InvalidSearchException from seqr.utils.search.elasticsearch.es_utils import InvalidIndexException @@ -903,7 +904,7 @@ def _assert_expected_single_variant_results_context(self, response_json, omit_fi @mock.patch('seqr.views.apis.variant_search_api.variant_lookup') def test_variant_lookup(self, mock_variant_lookup): response_variant = deepcopy(VARIANT_LOOKUP_VARIANT) - mock_variant_lookup.side_effect = lambda *args, **kwargs: deepcopy(response_variant) + mock_variant_lookup.side_effect = lambda *args, **kwargs: [deepcopy(response_variant)] url = f'{reverse(variant_lookup_handler)}?variantId=1-10439-AC-A&genomeVersion=38' self.check_require_login(url) @@ -936,13 +937,7 @@ def test_variant_lookup(self, mock_variant_lookup): }, } del expected_variant['familyGenotypes'] - expected_body = { - **{k: {} for k in self.EXPECTED_SEARCH_RESPONSE if k not in { - 'searchedVariants', 'search', 'variantNotesByGuid', 'variantTagsByGuid', 'variantFunctionalDataByGuid', 'transcriptsById', - }}, - **{k: {} for k in EXPECTED_SEARCH_FAMILY_CONTEXT}, - 'projectsByGuid': {}, - 'individualsByGuid': { + expected_body = self._expected_lookup_body({ 'I0_F0_1-10439-AC-A': { 'affected': 'N', 'familyGuid': 'F0_1-10439-AC-A', 'features': [], 'individualGuid': 'I0_F0_1-10439-AC-A', 'sex': 'F', @@ -969,12 +964,10 @@ def test_variant_lookup(self, mock_variant_lookup): 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', }, }, - 'variants': [expected_variant], - } - if 'totalSampleCounts' in self.EXPECTED_SEARCH_RESPONSE: - expected_body['totalSampleCounts'] = self.EXPECTED_SEARCH_RESPONSE['totalSampleCounts'] + [expected_variant], + ) self.assertDictEqual(response.json(), expected_body) - mock_variant_lookup.assert_called_with(self.no_access_user, ('1', 10439, 'AC', 'A'), genome_version='38') + mock_variant_lookup.assert_called_with(self.no_access_user, '1-10439-AC-A', '38', sample_type=None) response_variant['transcripts'] = VARIANTS[0]['transcripts'] expected_variant['transcripts'] = VARIANTS[0]['transcripts'] @@ -1006,36 +999,25 @@ def test_variant_lookup(self, mock_variant_lookup): 'liftedFamilyGuids': ['F000014_14'], 'genotypes': { individual_guid: [ - {**sample_genotype, **genotype} for sample_genotype in expected_variant['genotypes'][anon_individual_guid] + {**sample_genotype, **genotype, 'individualGuid': individual_guid} for sample_genotype in expected_variant['genotypes'][anon_individual_guid] ] if isinstance(expected_variant['genotypes'][anon_individual_guid], list) else { - **expected_variant['genotypes'][anon_individual_guid], **genotype, + **expected_variant['genotypes'][anon_individual_guid], **genotype, 'individualGuid': individual_guid, } for individual_guid, anon_individual_guid, genotype in individual_guid_map }, 'genomeVersion': '37', 'variantId': '1-248367227-TC-T', }) + expected_body.update(self._expected_lookup_context( + [PROJECT_GUID, 'R0003_test', 'R0004_non_analyst_project'], + ['F000002_2', 'F000011_11', 'F000014_14'], + [i[0] for i in individual_guid_map] + ['I000019_na21987', 'I000021_na21654']), + ) expected_body.update({ - **{k: {**EXPECTED_SEARCH_RESPONSE[k]} for k in { - 'mmeSubmissionsByGuid', 'variantTagsByGuid', 'variantNotesByGuid', - }}, - **EXPECTED_TRANSCRIPTS_RESPONSE, - 'omimIntervals': {}, - 'savedVariantsByGuid': {k: v for k, v in self.EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if k in ['SV0000002_1248367227_r0390_100']}, - 'variantFunctionalDataByGuid': {}, - 'locusListsByGuid': EXPECTED_SEARCH_CONTEXT_RESPONSE['locusListsByGuid'], - 'projectsByGuid': { - p: {k: mock.ANY for k in PROJECT_TAG_TYPE_FIELDS} - for p in [PROJECT_GUID, 'R0003_test', 'R0004_non_analyst_project'] - }, - 'familiesByGuid': { - f: {k: mock.ANY for k in [*FAMILY_FIELDS, 'individualGuids']} - for f in ['F000002_2', 'F000011_11', 'F000014_14'] - }, - 'individualsByGuid': { - i[0]: {k: mock.ANY for k in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']} - for i in individual_guid_map + [('I000019_na21987',), (('I000021_na21654',))] - }, + k: {**EXPECTED_SEARCH_RESPONSE[k]} for k in {'mmeSubmissionsByGuid', 'variantTagsByGuid', 'variantNotesByGuid'} }) + expected_body['savedVariantsByGuid']= { + k: v for k, v in self.EXPECTED_SEARCH_RESPONSE['savedVariantsByGuid'].items() if k in ['SV0000002_1248367227_r0390_100'] + } expected_body['genesById']['ENSG00000227232'] = expected_pa_gene expected_body['mmeSubmissionsByGuid']['MS000018_P0004517'] = expected_body['mmeSubmissionsByGuid'].pop('MS000001_na19675') expected_body['savedVariantsByGuid']['SV0000006_1248367227_r0004_non'] = mock.ANY @@ -1046,29 +1028,147 @@ def test_variant_lookup(self, mock_variant_lookup): self.assertDictEqual(response.json(), expected_body) mock_variant_lookup.assert_called_with( - self.manager_user, ('1', 10439, 'AC', 'A'), genome_version='37', + self.manager_user, '1-10439-AC-A', '37', sample_type=None, ) - @mock.patch('seqr.views.apis.variant_search_api.sv_variant_lookup') + def _expected_lookup_body(self, individuals_by_guid, variants): + expected_body = { + **{k: {} for k in self.EXPECTED_SEARCH_RESPONSE if k not in { + 'searchedVariants', 'search', 'variantNotesByGuid', 'variantTagsByGuid', 'variantFunctionalDataByGuid', + 'transcriptsById', + }}, + **{k: {} for k in EXPECTED_SEARCH_FAMILY_CONTEXT}, + 'projectsByGuid': {}, + 'individualsByGuid': individuals_by_guid, + 'variants': variants, + } + if 'totalSampleCounts' in self.EXPECTED_SEARCH_RESPONSE: + expected_body['totalSampleCounts'] = self.EXPECTED_SEARCH_RESPONSE['totalSampleCounts'] + return expected_body + + def _expected_lookup_context(self, project_guids, family_guids, individual_guids): + return { + **EXPECTED_TRANSCRIPTS_RESPONSE, + 'omimIntervals': {}, + 'variantFunctionalDataByGuid': {}, + 'locusListsByGuid': EXPECTED_SEARCH_CONTEXT_RESPONSE['locusListsByGuid'], + 'projectsByGuid': { + p: {k: mock.ANY for k in PROJECT_TAG_TYPE_FIELDS} for p in project_guids + }, + 'familiesByGuid': { + f: {k: mock.ANY for k in [*FAMILY_FIELDS, 'individualGuids']} for f in family_guids + }, + 'individualsByGuid': { + i: {k: mock.ANY for k in [*INDIVIDUAL_FIELDS, 'igvSampleGuids']} for i in individual_guids + }, + } + + @mock.patch('seqr.views.apis.variant_search_api.variant_lookup') def test_sv_variant_lookup(self, mock_variant_lookup): - variants = [{**variant, 'familyGuids': ['F000001_1']} for variant in [SV_VARIANT1, GCNV_VARIANT1]] - mock_variant_lookup.return_value = variants + lookup_variants = [ + {**GCNV_LOOKUP_VARIANT, 'familyGenotypes': {k: v for k, v in GCNV_LOOKUP_VARIANT['familyGenotypes'].items() if k != 'F000002_2_x'}}, + SV_LOOKUP_VARIANT, + ] + mock_variant_lookup.side_effect = lambda *args, **kwargs: [deepcopy(v) for v in lookup_variants] url = f'{reverse(variant_lookup_handler)}?variantId=phase2_DEL_chr14_4640&genomeVersion=37&sampleType=WGS' - self.check_collaborator_login(url) + self.check_require_login(url) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self._assert_expected_single_variant_results_context( - response.json(), variants=variants, omit_fields={'searchedVariants'}, no_metadata=True, - ) + expected_gcnv_variant = { + **GCNV_VARIANT4, + 'familyGuids': [], + 'lookupFamilyGuids': ['F0_suffix_140608_DUP'], + 'genotypes': { + 'I0_F0_suffix_140608_DUP': { + k: v for k, v in GCNV_VARIANT4['genotypes']['I000004_hg00731'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I1_F0_suffix_140608_DUP': { + k: v for k, v in GCNV_VARIANT4['genotypes']['I000005_hg00732'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I2_F0_suffix_140608_DUP': { + k: v for k, v in GCNV_VARIANT4['genotypes']['I000006_hg00733'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + }, + } + expected_sv_variant = { + **SV_VARIANT4, + 'familyGuids': [], + 'lookupFamilyGuids': ['F0_phase2_DEL_chr14_4640'], + 'genotypes': { + 'I0_F0_phase2_DEL_chr14_4640': { + k: v for k, v in SV_VARIANT4['genotypes']['I000018_na21234'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I1_F0_phase2_DEL_chr14_4640': { + k: v for k, v in SV_VARIANT4['genotypes']['I000019_na21987'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + 'I2_F0_phase2_DEL_chr14_4640': { + k: v for k, v in SV_VARIANT4['genotypes']['I000021_na21654'].items() + if k not in {'familyGuid', 'individualGuid', 'sampleId'} + }, + }, + } + expected_body = self._expected_lookup_body({ + 'I0_F0_phase2_DEL_chr14_4640': { + 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], + 'individualGuid': 'I0_F0_phase2_DEL_chr14_4640', 'sex': 'F', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I0_F0_suffix_140608_DUP': { + 'affected': 'A', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I0_F0_suffix_140608_DUP', + 'features': [{'category': 'HP:0000707', 'label': '1 terms'}, {'category': 'HP:0001626', 'label': '1 terms'}], + 'sex': 'X0', 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I1_F0_phase2_DEL_chr14_4640': { + 'affected': 'A', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], + 'individualGuid': 'I1_F0_phase2_DEL_chr14_4640', 'sex': 'M', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I1_F0_suffix_140608_DUP': { + 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I1_F0_suffix_140608_DUP', + 'sex': 'M','features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + 'I2_F0_phase2_DEL_chr14_4640': { + 'affected': 'N', 'familyGuid': 'F0_phase2_DEL_chr14_4640', 'features': [], + 'individualGuid': 'I2_F0_phase2_DEL_chr14_4640', 'sex': 'F', + 'vlmContactEmail': 'vlm@broadinstitute.org', + }, + 'I2_F0_suffix_140608_DUP': { + 'affected': 'N', 'familyGuid': 'F0_suffix_140608_DUP', 'individualGuid': 'I2_F0_suffix_140608_DUP', + 'sex': 'F', 'features': [], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', + }, + },[expected_gcnv_variant, expected_sv_variant]) + self.assertDictEqual(response.json(), expected_body) mock_variant_lookup.assert_called_with( - self.collaborator_user, 'phase2_DEL_chr14_4640', mock.ANY, genome_version='37', sample_type='WGS', + self.no_access_user, 'phase2_DEL_chr14_4640', '37', sample_type='WGS', ) - self.assertSetEqual({ - 'F000001_1', 'F000002_2', 'F000003_3', 'F000004_4', 'F000005_5', 'F000006_6', 'F000007_7', 'F000008_8', - 'F000009_9', 'F000010_10', 'F000013_13', - }, {f.guid for f in mock_variant_lookup.call_args.args[2]}) + + self.login_manager() + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + expected_gcnv_variant.update({ + 'lookupFamilyGuids': GCNV_VARIANT4['familyGuids'], + 'genotypes': GCNV_VARIANT4['genotypes'], + }) + expected_sv_variant.update({ + 'lookupFamilyGuids': SV_VARIANT4['familyGuids'], + 'genotypes': SV_VARIANT4['genotypes'], + }) + expected_body.update(self._expected_lookup_context( + [PROJECT_GUID, 'R0004_non_analyst_project'], + ['F000002_2', 'F000014_14'], + ['I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000018_na21234', 'I000019_na21987', 'I000021_na21654'], + )) + expected_body.update({k: {} for k in {'mmeSubmissionsByGuid', 'variantTagsByGuid', 'variantNotesByGuid'}}) + del expected_body['transcriptsById'] + self.assertDictEqual(response.json(), expected_body) + mock_variant_lookup.assert_called_with(self.manager_user, 'phase2_DEL_chr14_4640', '37', sample_type='WGS') @mock.patch('seqr.views.utils.vlm_utils.VLM_CLIENT_SECRET', 'abc123') @mock.patch('seqr.views.utils.vlm_utils.VLM_CLIENT_ID', MOCK_CLIENT_ID) diff --git a/seqr/views/react_app.py b/seqr/views/react_app.py index 84744cb5cd..10164a0679 100644 --- a/seqr/views/react_app.py +++ b/seqr/views/react_app.py @@ -60,7 +60,7 @@ def render_app_html(request, additional_json=None, include_user=True, status=200 'version': '{}-{}'.format(SEQR_VERSION, ui_version), 'hijakEnabled': DEBUG or False, 'oauthLoginProvider': SOCIAL_AUTH_PROVIDER, - 'elasticsearchEnabled': backend_specific_call(True, False, False), + 'elasticsearchEnabled': backend_specific_call(True, False), 'vlmEnabled': bool(VLM_CLIENT_ID), 'warningMessages': [message.json() for message in WarningMessage.objects.all()], 'anvilLoadingDelayDate': ANVIL_LOADING_DELAY_EMAIL_START_DATE if should_show_loading_delay else None, diff --git a/seqr/views/status.py b/seqr/views/status.py index 4775d94444..d0e6502a1e 100644 --- a/seqr/views/status.py +++ b/seqr/views/status.py @@ -17,7 +17,8 @@ def status_view(request): # Test database connection for db_connection_key in DATABASES.keys(): try: - connections[db_connection_key].cursor() + with connections[db_connection_key].cursor() as cursor: + cursor.execute('SELECT 1') except Exception as e: dependent_services_ok = False logger.error('Database "{}" connection error: {}'.format(db_connection_key, e)) diff --git a/seqr/views/status_tests.py b/seqr/views/status_tests.py index f96464d0ca..33518adef3 100644 --- a/seqr/views/status_tests.py +++ b/seqr/views/status_tests.py @@ -1,3 +1,4 @@ +from django.db import connections from django.test import TestCase from django.urls.base import reverse import mock @@ -8,42 +9,59 @@ @mock.patch('clickhouse_search.search.CLICKHOUSE_SERVICE_HOSTNAME', '') -class StatusTest(object): +@mock.patch('seqr.views.status.redis.StrictRedis') +@mock.patch('seqr.views.status.logger') +class ElasticsearchStatusTest(TestCase): + databases = '__all__' - def _test_status_error(self, url, mock_logger): + SEARCH_BACKEND_ERROR = 'No response from elasticsearch ping' + HAS_KIBANA = True + ES_HOSTNAME = 'testhost' + + def setUp(self): + patcher = mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', self.ES_HOSTNAME) + patcher.start() + self.addCleanup(patcher.stop) + + def _post_teardown(self): + for conn in connections.all(): + conn.connection = None + conn.ensure_connection() + if hasattr(conn.connection, 'pool'): + conn.connection.pool.closed = False + conn.connection.is_closed = False + + @urllib3_responses.activate + def test_status_error(self, mock_logger, mock_redis): + for conn in connections.all(): + conn.ensure_connection() + conn.connection.close() + mock_redis.return_value.ping.side_effect = HTTPError('Bad connection') + urllib3_responses.add(urllib3_responses.HEAD, '/status', status=400) + + url = reverse(status_view) response = self.client.get(url) self.assertEqual(response.status_code, 400) self.assertDictEqual( response.json(), {'version': 'v1.0', 'dependent_services_ok': False, 'secondary_services_ok': False}) calls = [ - mock.call('Database "default" connection error: No connection'), - mock.call('Database "reference_data" connection error: No connection'), - mock.call('Database "clickhouse_write" connection error: No connection'), - mock.call('Database "clickhouse" connection error: No connection'), + mock.call('Database "default" connection error: the connection is closed'), + mock.call('Database "reference_data" connection error: the connection is closed'), + mock.call('Database "clickhouse_write" connection error: connection already closed'), + mock.call('Database "clickhouse" connection error: connection already closed'), mock.call('Redis connection error: Bad connection'), - mock.call(f'Search backend connection error: {self.SEARCH_BACKEND_ERROR}'), ] + if self.SEARCH_BACKEND_ERROR: + calls.append(mock.call(f'Search backend connection error: {self.SEARCH_BACKEND_ERROR}')) if self.HAS_KIBANA: calls.append(mock.call('Search Admin connection error: Kibana Error 400: Bad Request')) mock_logger.error.assert_has_calls(calls) - mock_logger.reset_mock() - @mock.patch('seqr.views.status.redis.StrictRedis') - @mock.patch('seqr.views.status.connections') - @mock.patch('seqr.views.status.logger') @urllib3_responses.activate - def test_status(self, mock_logger, mock_db_connections, mock_redis): + def test_status(self, mock_logger, mock_redis): url = reverse(status_view) - mock_db_connections.__getitem__.return_value.cursor.side_effect = Exception('No connection') - mock_redis.return_value.ping.side_effect = HTTPError('Bad connection') - urllib3_responses.add(urllib3_responses.HEAD, '/status', status=400) - - self._test_status_error(url, mock_logger) - - mock_db_connections.__getitem__.return_value.cursor.side_effect = None mock_redis.return_value.ping.side_effect = None - urllib3_responses.reset() urllib3_responses.add(urllib3_responses.HEAD, '/', status=200) urllib3_responses.add(urllib3_responses.HEAD, '/status', status=500 if self.HAS_KIBANA else 200) @@ -67,30 +85,15 @@ def test_status(self, mock_logger, mock_db_connections, mock_redis): mock_logger.error.assert_not_called() self._assert_expected_requests() - -class ElasticsearchStatusTest(TestCase, StatusTest): - - SEARCH_BACKEND_ERROR = 'No response from elasticsearch ping' - HAS_KIBANA = True - - @mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', 'testhost') - def test_status(self, *args): - super(ElasticsearchStatusTest, self).test_status(*args) - def _assert_expected_requests(self): self.assertListEqual([call.request.url for call in urllib3_responses.calls], ['/', '/status', '/', '/status']) -class HailSearchStatusTest(TestCase, StatusTest): +class ClickhouseStatusTest(ElasticsearchStatusTest): - SEARCH_BACKEND_ERROR = '400: Bad Request' + SEARCH_BACKEND_ERROR = None HAS_KIBANA = False - - @mock.patch('seqr.utils.search.elasticsearch.es_utils.ELASTICSEARCH_SERVICE_HOSTNAME', '') - @mock.patch('seqr.utils.search.hail_search_utils.HAIL_BACKEND_SERVICE_HOSTNAME', 'test-hail') - def test_status(self, *args): - super(HailSearchStatusTest, self).test_status(*args) + ES_HOSTNAME = '' def _assert_expected_requests(self): - self.assertEqual(len(urllib3_responses.calls), 1) - self.assertEqual(urllib3_responses.calls[0].request.url, '/status') + self.assertEqual(len(urllib3_responses.calls), 0) diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 4bc6826e4d..ec867c784d 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -347,7 +347,7 @@ def _get_parsed_saved_discovery_variants_by_family( variants = [] gene_ids = set() variant_json_by_guid = backend_specific_call( - _get_variant_json_by_guid, _get_variant_json_by_guid, _get_clickhouse_variant_json_by_guid, + _get_variant_json_by_guid, _get_clickhouse_variant_json_by_guid, )(project_saved_variants, include_clinvar) for variant in project_saved_variants: chrom, pos = get_chrom_pos(variant.xpos) diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index 733de7657e..77b1a69ed3 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -203,7 +203,7 @@ def check_project_individuals_deletable(project, individual_guids=None): if individual_guids is not None: individuals_to_delete = individuals_to_delete.filter(guid__in=individual_guids) - errors = backend_specific_call(_validate_no_submissions, _validate_no_sumissions_no_search_samples, _validate_no_sumissions_no_search_samples)(individuals_to_delete) + errors = backend_specific_call(_validate_no_submissions, _validate_no_sumissions_no_search_samples)(individuals_to_delete) return errors, individuals_to_delete diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 1f6ecb75dc..05b8c96239 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -424,7 +424,6 @@ def get_json_for_saved_variants(saved_variants, add_details=False, additional_mo if add_details: from seqr.utils.search.utils import backend_specific_call additional_fields += ['saved_variant_json'] + backend_specific_call( - lambda x: [], lambda x: [], lambda gv: ['key', 'genotypes', 'dataset_type'] + ([] if gv else ['family__project__genome_version']), )(genome_version) @@ -436,7 +435,7 @@ def get_json_for_saved_variants(saved_variants, add_details=False, additional_mo if add_details: from seqr.utils.search.utils import backend_specific_call - backend_specific_call(lambda *args: None, lambda *args: None, _add_clickhouse_annotations)(results, genome_version) + backend_specific_call(lambda *args: None, _add_clickhouse_annotations)(results, genome_version) for result in results: result.update({k: v for k, v in result.pop('savedVariantJson').items() if k not in result}) @@ -524,10 +523,10 @@ def get_json_for_saved_variants_child_entities(tag_cls, saved_variant_id_map, ta return tags, variant_tag_map -def get_json_for_saved_variants_with_tags(saved_variants, **kwargs): +def get_json_for_saved_variants_with_tags(saved_variants, additional_model_fields=None, **kwargs): variants_by_guid = { variant['variantGuid']: dict(tagGuids=[], functionalDataGuids=[], noteGuids=[], **variant) - for variant in get_json_for_saved_variants(saved_variants, additional_model_fields=['id'], **kwargs) + for variant in get_json_for_saved_variants(saved_variants, additional_model_fields=(additional_model_fields or []) + ['id'], **kwargs) } saved_variant_id_map = {} diff --git a/seqr/views/utils/project_context_utils.py b/seqr/views/utils/project_context_utils.py index 98319a2392..82121c696f 100644 --- a/seqr/views/utils/project_context_utils.py +++ b/seqr/views/utils/project_context_utils.py @@ -1,12 +1,17 @@ from collections import defaultdict from django.db.models import Count, Q, F, prefetch_related_objects +from clickhouse_search.search import get_transcripts_by_key from seqr.models import Individual, IgvSample, AnalysisGroup, DynamicAnalysisGroup, LocusList, VariantTagType,\ VariantFunctionalData, FamilyNote, SavedVariant, VariantTag, VariantNote from seqr.utils.gene_utils import get_genes +from seqr.utils.logging_utils import SeqrLogger +from seqr.utils.search.utils import backend_specific_call from seqr.views.utils.orm_to_json_utils import _get_json_for_families, _get_json_for_individuals, get_json_for_queryset, \ get_json_for_analysis_groups, get_json_for_samples, get_json_for_locus_lists, \ - get_json_for_family_notes, get_json_for_saved_variants + get_json_for_family_notes + +logger = SeqrLogger(__name__) def get_projects_child_entities(projects, project_guid, user): @@ -114,15 +119,22 @@ def families_discovery_tags(families, genome_version, project=None): families_by_guid = {f['familyGuid']: dict(discoveryTags=[], **f) for f in families} family_filter = {'family__project': project} if project else {'family__guid__in': families_by_guid.keys()} - discovery_tags = get_json_for_saved_variants(SavedVariant.objects.filter( + discovery_tags_by_key = {v.pop('key'): v for v in SavedVariant.objects.filter( varianttag__variant_tag_type__category='CMG Discovery Tags', **family_filter, - ), add_details=True, genome_version=genome_version) + ).values( + 'key', 'family__guid', selectedMainTranscriptId=F('selected_main_transcript_id'), + transcripts=F('saved_variant_json__transcripts'), mainTranscriptId=F('saved_variant_json__mainTranscriptId'), + )} + try: + backend_specific_call(lambda *args: None, _add_clickhouse_transcripts)(discovery_tags_by_key, genome_version) + except Exception as e: + logger.error(f'Error loading discovery genes from clickhouse: {e}', None) gene_ids = set() - for tag in discovery_tags: - gene_ids.update(list(tag.get('transcripts', {}).keys())) - for family_guid in tag['familyGuids']: - families_by_guid[family_guid]['discoveryTags'].append(tag) + for tag in discovery_tags_by_key.values(): + tag['transcripts'] = tag.get('transcripts') or {} + gene_ids.update(list(tag['transcripts'].keys())) + families_by_guid[tag.pop('family__guid')]['discoveryTags'].append(tag) return { 'familiesByGuid': families_by_guid, @@ -130,6 +142,17 @@ def families_discovery_tags(families, genome_version, project=None): } +def _add_clickhouse_transcripts(discovery_tags_by_key, genome_version): + transcripts_by_key = get_transcripts_by_key(genome_version, discovery_tags_by_key.keys()) + for key, tag in discovery_tags_by_key.items(): + if key in transcripts_by_key: + tag['transcripts'] = transcripts_by_key[key] + tag['mainTranscriptId'] = next( + t['transcriptId'] for gene_transcripts in tag['transcripts'].values() for t in gene_transcripts + if t['transcriptRank'] == 0 + ) + + MME_TAG_NAME = 'MME Submission' diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 5f71832f7a..a1663e3f65 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -15,7 +15,8 @@ from reference_data.models import TranscriptInfo, Omim, GENOME_VERSION_GRCh38 from seqr.models import SavedVariant, VariantSearchResults, Family, LocusList, LocusListInterval, LocusListGene, \ RnaSeqTpm, PhenotypePrioritization, Project, Sample, RnaSample, VariantTag, VariantTagType -from seqr.utils.search.utils import get_variants_for_variant_ids, backend_specific_call, parse_variant_id +from seqr.utils.search.elasticsearch.es_utils import get_es_variants_for_variant_ids +from seqr.utils.search.utils import backend_specific_call, parse_variant_id from seqr.utils.gene_utils import get_genes_for_variants from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.xpos_utils import get_xpos @@ -30,20 +31,18 @@ logger = logging.getLogger(__name__) -MAX_VARIANTS_FETCH = 1000 DISCOVERY_CATEGORY = 'CMG Discovery Tags' OMIM_GENOME_VERSION = GENOME_VERSION_GRCh38 -def update_projects_saved_variant_json(projects, user_email, update_function=None, **kwargs): +def update_projects_saved_variant_json(projects, user_email, update_function, **kwargs): success = {} skipped = {} error = {} - updated_variants_by_id = {} logger.info(f'Reloading saved variants in {len(projects)} projects') for project_id, project_guid, project_name, genome_version, family_guids in tqdm(projects, unit=' project'): try: - updated_saved_variants = (update_function or update_project_saved_variant_json)( + updated_saved_variants = update_function( project_id, genome_version, user_email=user_email, family_guids=family_guids, project_guid=project_guid, **kwargs) if updated_saved_variants is None: skipped[project_name] = True @@ -51,7 +50,6 @@ def update_projects_saved_variant_json(projects, user_email, update_function=Non success[project_name] = len(updated_saved_variants) family_summary = f' in {len(family_guids)} families' if family_guids else '' logger.info(f'Updated {len(updated_saved_variants)} variants{family_summary} for project {project_name}') - updated_variants_by_id.update({v.variant_id: v.saved_variant_json for v in updated_saved_variants.values()}) except Exception as e: traceback_message = traceback.format_exc() logger.error(traceback_message) @@ -68,7 +66,6 @@ def update_projects_saved_variant_json(projects, user_email, update_function=Non logger.info(f'{len(error)} failed projects') for k, v in error.items(): logger.info(f' {k}: {v}') - return updated_variants_by_id def get_saved_variants(genome_version, project_id=None, family_guids=None, dataset_type=None, clickhouse_dataset_type=None): @@ -87,38 +84,6 @@ def get_saved_variants(genome_version, project_id=None, family_guids=None, datas return saved_variants -def update_project_saved_variant_json(project_id, genome_version, family_guids=None, dataset_type=None, user=None, user_email=None, **kwargs): - saved_variants = get_saved_variants(genome_version, project_id, family_guids, dataset_type).select_related('family') - - if not saved_variants: - return None - - families = set() - variant_ids = set() - saved_variants_map = {} - for v in saved_variants: - families.add(v.family) - variant_ids.add(v.variant_id) - saved_variants_map[(v.variant_id, v.family.guid)] = v - - variant_ids = sorted(variant_ids) - families = sorted(families, key=lambda f: f.guid) - variants_json = [] - for sub_var_ids in [variant_ids[i:i+MAX_VARIANTS_FETCH] for i in range(0, len(variant_ids), MAX_VARIANTS_FETCH)]: - variants_json += get_variants_for_variant_ids(families, sub_var_ids, user=user, user_email=user_email) - - updated_saved_variants = {} - for var in variants_json: - for family_guid in var['familyGuids']: - saved_variant = saved_variants_map.get((var['variantId'], family_guid)) - if saved_variant: - saved_variant.saved_variant_json = var - updated_saved_variants[saved_variant.guid] = saved_variant - SavedVariant.bulk_update_models(user, list(updated_saved_variants.values()), ['saved_variant_json']) - - return updated_saved_variants - - def saved_variants_dataset_type_filter(dataset_type): xpos_filter_key = 'xpos__gte' if dataset_type == Sample.DATASET_TYPE_MITO_CALLS else 'xpos__lt' dataset_filter = {xpos_filter_key: get_xpos('M', 1)} @@ -177,7 +142,7 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use new_variant_keys = set(family_variant_data.keys()) - set(saved_variant_map.keys()) if new_variant_keys: new_variant_data = _search_new_saved_variants(new_variant_keys, user, genome_version) if load_new_variant_data else backend_specific_call( - lambda o, _: o, lambda o, _: o, _get_clickhouse_variant_keys, + lambda o, _: o, _get_clickhouse_variant_keys, )({k: v for k, v in family_variant_data.items() if k in new_variant_keys}, genome_version) new_variant_models = [] for (family_id, variant_id), variant in new_variant_data.items(): @@ -196,7 +161,7 @@ def bulk_create_tagged_variants(family_variant_data, tag_name, get_metadata, use } variant_genes_by_id = backend_specific_call( - _get_saved_variant_genes, _get_saved_variant_genes, _get_clickhouse_saved_variant_genes, + _get_saved_variant_genes, _get_clickhouse_saved_variant_genes, )(saved_variant_map.values(), genome_version) update_tags = [] @@ -281,10 +246,11 @@ def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], user: U variant_families[variant_id].append(family_id) families_by_id = {f.id: f for f in Family.objects.filter(id__in=family_ids)} + samples = Sample.objects.filter(is_active=True, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) search_variants_by_id = { v['variantId']: v for v in backend_specific_call( - get_variants_for_variant_ids, get_variants_for_variant_ids, _get_clickhouse_variants, - )(families=list(families_by_id.values()), variant_ids=list(variant_families.keys()), user=user, genome_version=genome_version) + _get_es_variants, _get_clickhouse_variants, + )(samples, families_by_id=families_by_id, variant_ids=list(variant_families.keys()), user=user, genome_version=genome_version) } new_variants = {} @@ -307,14 +273,18 @@ def _search_new_saved_variants(family_variant_ids: set[tuple[int, str]], user: U return new_variants -def _get_clickhouse_variants(families: list[Family], variant_ids: list[str], genome_version: str = None, **kwargs) -> list[dict]: +def _get_es_variants(samples: Sample.objects, families_by_id: dict[int, Family], *args, **kwargs): + return get_es_variants_for_variant_ids(samples.filter(individual__family_id__in=families_by_id.keys()), *args, **kwargs) + + +def _get_clickhouse_variants(samples: Sample.objects, families_by_id: dict[int, Family], variant_ids: list[str], genome_version: str = None, **kwargs) -> list[dict]: variant_key_map = get_clickhouse_key_lookup(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variant_ids, reverse=True) + families = list(families_by_id.values()) prefetch_related_objects(families, 'project') families_by_project = defaultdict(list) for family in families: families_by_project[family.project.guid].append(family.guid) variants = [] - samples = Sample.objects.filter(is_active=True, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) for project_guid, family_guids in families_by_project.items(): genotype_keys = get_clickhouse_genotypes( project_guid, family_guids, genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variant_key_map.keys(), samples, @@ -380,7 +350,7 @@ def _saved_variant_genes_transcripts(variants): for var in variant: for gene_id, transcripts in var.get('transcripts', {}).items(): gene_ids.add(gene_id) - if backend_specific_call(lambda v: True, _requires_transcript_metadata, _requires_transcript_metadata)(variant): + if backend_specific_call(lambda v: True, _requires_transcript_metadata)(variant): transcript_ids.update([t['transcriptId'] for t in transcripts if t.get('transcriptId')]) for family_guid in var['familyGuids']: family_genes[family_guid].update(var.get('transcripts', {}).keys()) @@ -556,7 +526,7 @@ def get_variants_response(request, saved_variants, response_variants=None, add_a response['familiesByGuid'][family_guid] = {} response['familiesByGuid'][family_guid].update(data) - backend_specific_call(lambda response: response, _add_sample_count_stats, _add_sample_count_stats)(response) + backend_specific_call(lambda response: response, _add_sample_count_stats)(response) return response diff --git a/settings.py b/settings.py index 891a8a030d..eab7c3bb77 100644 --- a/settings.py +++ b/settings.py @@ -374,9 +374,6 @@ KIBANA_ELASTICSEARCH_USER = os.environ.get('KIBANA_ELASTICSEARCH_USER', 'kibana') KIBANA_ELASTICSEARCH_PASSWORD = os.environ.get('KIBANA_ES_PASSWORD') -HAIL_BACKEND_SERVICE_HOSTNAME = os.environ.get('HAIL_BACKEND_SERVICE_HOSTNAME', 'hail-search') -HAIL_BACKEND_SERVICE_PORT = int(os.environ.get('HAIL_BACKEND_SERVICE_PORT', '5000')) - REDIS_SERVICE_HOSTNAME = os.environ.get('REDIS_SERVICE_HOSTNAME', 'localhost') REDIS_SERVICE_PORT = int(os.environ.get('REDIS_SERVICE_PORT', '6379')) diff --git a/ui/package.json b/ui/package.json index 16d07b045a..9d9dabe99d 100644 --- a/ui/package.json +++ b/ui/package.json @@ -121,6 +121,7 @@ "start": "node scripts/start.js", "build": "node scripts/build.js", "test": "node scripts/test.js --env=jsdom", + "lint": "eslint --ext .js,.jsx --ignore-pattern \"**/*.test.js\" --ignore-pattern \"**/fixtures.js\" --ignore-pattern \"scripts/*\" .", "postinstall": "patch-package" }, "jest": { diff --git a/ui/pages/Project/reducers.js b/ui/pages/Project/reducers.js index d77a5a923c..9f783e1b98 100644 --- a/ui/pages/Project/reducers.js +++ b/ui/pages/Project/reducers.js @@ -256,13 +256,9 @@ export const getMmeMatches = submissionGuid => (dispatch, getState) => { const state = getState() const submission = state.mmeSubmissionsByGuid[submissionGuid] if (submission && !submission.geneVariants) { - const { familyGuid } = state.individualsByGuid[submission.individualGuid] dispatch({ type: REQUEST_MME_MATCHES }) new HttpRequestHelper(`/api/matchmaker/get_mme_matches/${submissionGuid}`, (responseJson) => { - dispatch({ - type: RECEIVE_SAVED_VARIANT_FAMILIES, updates: { [familyGuid]: { loaded: true, noteVariants: true } }, - }) dispatch({ type: RECEIVE_DATA, updatesById: responseJson, diff --git a/ui/pages/SummaryData/components/VariantLookup.jsx b/ui/pages/SummaryData/components/VariantLookup.jsx index d23c82b86d..820bb9691d 100644 --- a/ui/pages/SummaryData/components/VariantLookup.jsx +++ b/ui/pages/SummaryData/components/VariantLookup.jsx @@ -194,10 +194,10 @@ const VariantLookup = ({ queryParams, receiveData, updateQueryParams, vlmEnabled {vlmEnabled && ( - -
+ +
- + )} + {vlmEnabled && ( + + +
+ + + + )} - - { - info && info.length > 0 && - - {info.map(infoItem =>
{infoItem}
)} - - } - { - warnings && warnings.length > 0 && - - {warnings.map(warning =>
WARNING: {warning}
)} - - } - { - errors && errors.length > 0 && - - {errors.map(error =>
ERROR: {error}
)} - - } - , -) - -MessagesPanel.propTypes = { - errors: PropTypes.array, - warnings: PropTypes.array, - info: PropTypes.array, -} - -export default MessagesPanel diff --git a/ui/shared/components/page/BreadCrumbs.jsx b/ui/shared/components/page/BreadCrumbs.jsx deleted file mode 100644 index d342dcb310..0000000000 --- a/ui/shared/components/page/BreadCrumbs.jsx +++ /dev/null @@ -1,36 +0,0 @@ -/* eslint-disable react/no-array-index-key */ - -import React from 'react' -import PropTypes from 'prop-types' - -import { Breadcrumb } from 'semantic-ui-react' - -const BreadCrumbs = React.memo(props => -
- { - props.breadcrumbSections.map((label, i) => ( - { - i < props.breadcrumbSections.length - 1 ? - ( - - {props.breadcrumbSections[i]} - ยป - - ) : ( - - {props.breadcrumbSections[i]} - - ) - } - ), - ) - } - -
, -) - -BreadCrumbs.propTypes = { - breadcrumbSections: PropTypes.array.isRequired, -} - -export default BreadCrumbs diff --git a/ui/shared/components/page/BreadCrumbs.test.js b/ui/shared/components/page/BreadCrumbs.test.js deleted file mode 100644 index 6ba1fcfeb8..0000000000 --- a/ui/shared/components/page/BreadCrumbs.test.js +++ /dev/null @@ -1,18 +0,0 @@ -import React from 'react' -import { shallow, configure } from 'enzyme' -import Adapter from '@wojtekmaj/enzyme-adapter-react-17' -import BreadCrumbs from './BreadCrumbs' - -configure({ adapter: new Adapter() }) - -test('shallow-render without crashing', () => { - /* - breadcrumbSections: PropTypes.array.isRequired, - */ - - const props = { - breadcrumbSections: ['base', 'subsection', 'subsection2'], - } - - shallow() -}) diff --git a/vlm/hail_backend_utils.py b/vlm/hail_backend_utils.py deleted file mode 100644 index 6d12287870..0000000000 --- a/vlm/hail_backend_utils.py +++ /dev/null @@ -1,15 +0,0 @@ -import hail as hl -import os - -VLM_DATA_DIR = os.environ.get('VLM_DATA_DIR') - - -def get_hail_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, genome_build: str) -> hl.Struct: - interval = hl.eval(hl.interval(locus, locus, includes_start=True, includes_end=True)) - ht = hl.read_table( - f'{VLM_DATA_DIR}/{genome_build}/SNV_INDEL/annotations.ht', _intervals=[interval], _filter_intervals=True, - ) - ht = ht.filter(ht.alleles == hl.array([ref, alt])) - - counts = ht.aggregate(hl.agg.take(ht.gt_stats, 1)) - return (counts[0].AC, counts[0].hom) if counts else (0, 0) diff --git a/vlm/match.py b/vlm/match.py index 0795e0f87b..8c2b42cd62 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -3,9 +3,7 @@ import os from vlm.clickhouse_utils import get_clickhouse_variant_counts -from vlm.hail_backend_utils import get_hail_variant_counts -CLICKHOUSE_ENABLED = bool(os.environ.get('CLICKHOUSE_SERVICE_HOSTNAME')) SEQR_BASE_URL = os.environ.get('SEQR_BASE_URL') VLM_DEFAULT_CONTACT_EMAIL = os.environ.get('VLM_DEFAULT_CONTACT_EMAIL') NODE_ID = os.environ.get('NODE_ID') @@ -42,12 +40,11 @@ def get_variant_match(query: dict) -> dict: chrom, pos, ref, alt, genome_build = _parse_match_query(query) locus = hl.locus(chrom, pos, reference_genome=genome_build) - get_counts_func = get_clickhouse_variant_counts if CLICKHOUSE_ENABLED else get_hail_variant_counts - ac, hom = get_counts_func(locus, ref, alt, genome_build) + ac, hom = get_clickhouse_variant_counts(locus, ref, alt, genome_build) liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 liftover_locus = hl.liftover(locus, liftover_genome_build) - lift_ac, lift_hom = get_counts_func(liftover_locus, ref, alt, liftover_genome_build) + lift_ac, lift_hom = get_clickhouse_variant_counts(liftover_locus, ref, alt, liftover_genome_build) url = _get_contact_url( chrom, pos, ref, alt, genome_build, liftover_genome_build, liftover_locus if lift_ac and not ac else None,