From aec0e6f30300076a7d12837fa4040b7a78f8abb2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 10:34:51 -0400 Subject: [PATCH 01/46] use clickhosue for vlm --- vlm/deploy/Dockerfile | 2 ++ vlm/match.py | 27 ++++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/vlm/deploy/Dockerfile b/vlm/deploy/Dockerfile index 7788d5cd0b..821573ffad 100644 --- a/vlm/deploy/Dockerfile +++ b/vlm/deploy/Dockerfile @@ -2,6 +2,8 @@ FROM hailgenetics/hail:0.2.128 LABEL maintainer="Broad TGG" +RUN pip install clickhouse-connect==0.8.18 + WORKDIR /vlm # Application Code diff --git a/vlm/match.py b/vlm/match.py index e5dc38e8b1..6c15750465 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -1,4 +1,5 @@ from aiohttp.web import HTTPBadRequest +import clickhouse_connect import hail as hl import os @@ -7,6 +8,15 @@ VLM_DEFAULT_CONTACT_EMAIL = os.environ.get('VLM_DEFAULT_CONTACT_EMAIL') NODE_ID = os.environ.get('NODE_ID') +CLICKHOUSE_SERVICE_HOSTNAME = os.environ.get('CLICKHOUSE_SERVICE_HOSTNAME') +CLICKHOUSE_CONNECTION_PARAMS = { + 'host': CLICKHOUSE_SERVICE_HOSTNAME, + 'port': os.environ.get('CLICKHOUSE_SERVICE_PORT'), + 'username': os.environ.get('CLICKHOUSE_USERNAME'), + 'password': os.environ.get('CLICKHOUSE_PASSWORD'), + 'database': os.environ.get('CLICKHOUSE_DATABASE', 'seqr'), +} + BEACON_HANDOVER_TYPE = { 'id': NODE_ID, 'label': f'{NODE_ID} browser' @@ -38,11 +48,12 @@ def get_variant_match(query: dict) -> dict: chrom, pos, ref, alt, genome_build = _parse_match_query(query) locus = hl.locus(chrom, pos, reference_genome=genome_build) - ac, hom = _get_variant_counts(locus, ref, alt, genome_build) + get_counts_func = _get_clickhouse_variant_counts if CLICKHOUSE_SERVICE_HOSTNAME else _get_hail_variant_counts + ac, hom = get_counts_func(locus, ref, alt, genome_build) liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 liftover_locus = hl.liftover(locus, liftover_genome_build) - lift_ac, lift_hom = _get_variant_counts(liftover_locus, ref, alt, liftover_genome_build) + lift_ac, lift_hom = get_counts_func(liftover_locus, ref, alt, liftover_genome_build) url = _get_contact_url( chrom, pos, ref, alt, genome_build, liftover_genome_build, liftover_locus if lift_ac and not ac else None, @@ -88,7 +99,7 @@ def _parse_match_query(query: dict) -> tuple[str, int, str, str, str]: return chrom, start, query['referenceBases'], query['alternateBases'], genome_build -def _get_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, genome_build: str) -> hl.Struct: +def _get_hail_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, genome_build: str) -> hl.Struct: interval = hl.eval(hl.interval(locus, locus, includes_start=True, includes_end=True)) ht = hl.read_table( f'{VLM_DATA_DIR}/{genome_build}/SNV_INDEL/annotations.ht', _intervals=[interval], _filter_intervals=True, @@ -99,6 +110,16 @@ def _get_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, genome_bu return (counts[0].AC, counts[0].hom) if counts else (0, 0) +def _get_clickhouse_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, genome_build: str) -> hl.Struct: + locus = hl.eval(locus) + client = clickhouse_connect.get_client(**CLICKHOUSE_CONNECTION_PARAMS) + results = client.query( + f"SELECT dictGet(`{genome_build}/SNV_INDEL/gt_stats_dict`, ('ac_wes', 'ac_wgs', 'hom_wes', 'hom_wgs'), key) FROM `{genome_build}/SNV_INDEL/key_lookup` WHERE variantId=%(variant_id)s", + parameters={'variant_id': f'{locus.contig.replace("chr", "")}-{locus.position}-{ref}-{alt}'}, + ).result_set + return (results[0]['ac_wes'] + results[0]['ac_wgs'], results[0]['hom_wes'] + results[0]['hom_wgs']) if results else (0, 0) + + def _format_results(ac: int, hom: int, url: str) -> dict: total = ac - hom # Homozygotes count twice toward the total AC result_sets = [ From ff531224e0e958c8cec467497091649bf4c46095 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 11:16:43 -0400 Subject: [PATCH 02/46] clickhouse vlm unit tests --- .github/workflows/vlm-unit-tests.yaml | 54 ++++++++++++++++++++++++++- vlm/requirements-test.in | 1 + vlm/requirements-test.txt | 2 + 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index d83a7e0d80..c6ee5e6532 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -19,7 +19,7 @@ on: - 'hail_search/fixtures/*' jobs: - vlm: + vlm_hail_backend: runs-on: ubuntu-latest container: hailgenetics/hail:0.2.128 @@ -38,3 +38,55 @@ jobs: export JAVA_OPTS_XSS=16M coverage run --source="./vlm" --omit="./vlm/__main__.py" -m pytest vlm/ coverage report --fail-under=95 + + vlm_clickhouse: + runs-on: ubuntu-latest + container: hailgenetics/hail:0.2.128 + + services: + clickhouse: + image: bitnami/clickhouse:latest + ports: + - 8123:8123 # HTTP interface + options: >- + --health-cmd "clickhouse-client --query 'SELECT 1'" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + --name clickhouse + env: + CLICKHOUSE_USER: clickhouse_test + CLICKHOUSE_PASSWORD: clickhouse_test + ALLOW_EMPTY_PASSWORD: no + + steps: + - uses: actions/checkout@v2 + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip wheel + pip install -r vlm/requirements-test.txt + - name: Set up Clickhouse Settings and Data + run: | + docker exec clickhouse clickhouse-client --query "CREATE DATABASE seqr" + docker exec clickhouse clickhouse-client --query "CREATE TABLE seqr.`GRCh37/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId" + docker exec clickhouse clickhouse-client --query "INSERT INTO seqr.`GRCh37/SNV_INDEL/key_lookup` (*) VALUES ('7-143270172-A-G', 1), ('1-39190091-T-G', 2)" + docker exec clickhouse clickhouse-client --query "CREATE DICTIONARY seqr.`GRCh37/SNV_INDEL/gt_stats_dict` (key UInt32, ac_wes UInt32, ac_wgs UInt32, hom_wes UInt32, hom_wgs UInt32) PRIMARY KEY key SOURCE(CLICKHOUSE(USER 'clickhouse_test' PASSWORD 'clickhouse_test' QUERY `SELECT * FROM VALUES ((1, 4104, 607, 1276, 232), (2, 7, 2, 2, 1))`)) LIFETIME(0) LAYOUT(FLAT(MAX_ARRAY_SIZE 500000000))" + docker exec clickhouse clickhouse-client --query "CREATE TABLE seqr.`GRCh38/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId" + docker exec clickhouse clickhouse-client --query "INSERT INTO seqr.`GRCh38/SNV_INDEL/key_lookup` (*) VALUES ('1-38724419-T-G', 1)" + docker exec clickhouse clickhouse-client --query "CREATE DICTIONARY seqr.`GRCh38/SNV_INDEL/gt_stats_dict` (key UInt32, ac_wes UInt32, ac_wgs UInt32, hom_wes UInt32, hom_wgs UInt32) PRIMARY KEY key SOURCE(CLICKHOUSE(USER 'clickhouse_test' PASSWORD 'clickhouse_test' QUERY `SELECT * FROM VALUES ((1, 18, 10, 3, 1))`)) LIFETIME(0) LAYOUT(FLAT(MAX_ARRAY_SIZE 500000000))" + docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" + docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" + - name: Run coverage tests + run: | + export SEQR_BASE_URL=https://test-seqr.org/ + export NODE_ID=TestVLM + export CLICKHOUSE_SERVICE_HOSTNAME=localhost + export CLICKHOUSE_SERVICE_PORT=8123 + export CLICKHOUSE_USERNAME=vlm_test_user + export CLICKHOUSE_PASSWORD=vlm_test_password + coverage run --source="./vlm" --omit="./vlm/__main__.py" -m pytest vlm/ + coverage report --fail-under=95 + diff --git a/vlm/requirements-test.in b/vlm/requirements-test.in index f335034028..c63f835ebe 100644 --- a/vlm/requirements-test.in +++ b/vlm/requirements-test.in @@ -1,3 +1,4 @@ aioresponses coverage<5.2 pytest-aiohttp +clickhouse-connect==0.8.18 diff --git a/vlm/requirements-test.txt b/vlm/requirements-test.txt index df2cb3b32d..ccd84ac4f7 100644 --- a/vlm/requirements-test.txt +++ b/vlm/requirements-test.txt @@ -18,6 +18,8 @@ async-timeout==4.0.2 # via aiohttp attrs==23.1.0 # via aiohttp +clickhouse-connect==0.8.18 + # via -r vlm/requirements-test.in coverage==5.1 # via -r vlm/requirements-test.in exceptiongroup==1.1.3 From a7868013e57ae3a346bbecf301114a7dc62db4f0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 11:19:00 -0400 Subject: [PATCH 03/46] docker lint --- vlm/deploy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vlm/deploy/Dockerfile b/vlm/deploy/Dockerfile index 821573ffad..bda44c60eb 100644 --- a/vlm/deploy/Dockerfile +++ b/vlm/deploy/Dockerfile @@ -2,7 +2,7 @@ FROM hailgenetics/hail:0.2.128 LABEL maintainer="Broad TGG" -RUN pip install clickhouse-connect==0.8.18 +RUN pip install --no-cache-dir clickhouse-connect==0.8.18 WORKDIR /vlm From 068c5b7d549adf99bb2a85b916e30f5db0c05ef4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 11:23:08 -0400 Subject: [PATCH 04/46] code ql fix --- .github/workflows/vlm-unit-tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index c6ee5e6532..9a10af0fd6 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -1,4 +1,6 @@ name: VLM Unit Tests +permissions: + contents: read # Run the test suite on pushes (incl. merges) to master and dev # Run the test suite when a PR is opened, pushed to, or reopened From 7beb42393c8ea122be0c6a9be53694e9d7c8c175 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 11:52:17 -0400 Subject: [PATCH 05/46] python script for db setup --- .github/workflows/vlm-unit-tests.yaml | 20 +++--------- .../setup_clickhouse_test_data.py | 31 +++++++++++++++++++ 2 files changed, 36 insertions(+), 15 deletions(-) create mode 100644 vlm/liftover_references/setup_clickhouse_test_data.py diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 9a10af0fd6..f44e97f801 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -49,6 +49,7 @@ jobs: clickhouse: image: bitnami/clickhouse:latest ports: + - 9000:9000 # Native client interface - 8123:8123 # HTTP interface options: >- --health-cmd "clickhouse-client --query 'SELECT 1'" @@ -57,8 +58,8 @@ jobs: --health-retries 5 --name clickhouse env: - CLICKHOUSE_USER: clickhouse_test - CLICKHOUSE_PASSWORD: clickhouse_test + CLICKHOUSE_USER: clickhouse_test_user + CLICKHOUSE_PASSWORD: clickhouse_test_password ALLOW_EMPTY_PASSWORD: no steps: @@ -68,19 +69,7 @@ jobs: python3 -m pip install --upgrade pip wheel pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Settings and Data - run: | - docker exec clickhouse clickhouse-client --query "CREATE DATABASE seqr" - docker exec clickhouse clickhouse-client --query "CREATE TABLE seqr.`GRCh37/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId" - docker exec clickhouse clickhouse-client --query "INSERT INTO seqr.`GRCh37/SNV_INDEL/key_lookup` (*) VALUES ('7-143270172-A-G', 1), ('1-39190091-T-G', 2)" - docker exec clickhouse clickhouse-client --query "CREATE DICTIONARY seqr.`GRCh37/SNV_INDEL/gt_stats_dict` (key UInt32, ac_wes UInt32, ac_wgs UInt32, hom_wes UInt32, hom_wgs UInt32) PRIMARY KEY key SOURCE(CLICKHOUSE(USER 'clickhouse_test' PASSWORD 'clickhouse_test' QUERY `SELECT * FROM VALUES ((1, 4104, 607, 1276, 232), (2, 7, 2, 2, 1))`)) LIFETIME(0) LAYOUT(FLAT(MAX_ARRAY_SIZE 500000000))" - docker exec clickhouse clickhouse-client --query "CREATE TABLE seqr.`GRCh38/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId" - docker exec clickhouse clickhouse-client --query "INSERT INTO seqr.`GRCh38/SNV_INDEL/key_lookup` (*) VALUES ('1-38724419-T-G', 1)" - docker exec clickhouse clickhouse-client --query "CREATE DICTIONARY seqr.`GRCh38/SNV_INDEL/gt_stats_dict` (key UInt32, ac_wes UInt32, ac_wgs UInt32, hom_wes UInt32, hom_wgs UInt32) PRIMARY KEY key SOURCE(CLICKHOUSE(USER 'clickhouse_test' PASSWORD 'clickhouse_test' QUERY `SELECT * FROM VALUES ((1, 18, 10, 3, 1))`)) LIFETIME(0) LAYOUT(FLAT(MAX_ARRAY_SIZE 500000000))" - docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" - docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" + run: python3 vlm/setup_clickhouse_test_data.py clickhouse_test_user clickhouse_test_password - name: Run coverage tests run: | export SEQR_BASE_URL=https://test-seqr.org/ @@ -89,6 +78,7 @@ jobs: export CLICKHOUSE_SERVICE_PORT=8123 export CLICKHOUSE_USERNAME=vlm_test_user export CLICKHOUSE_PASSWORD=vlm_test_password + export CLICKHOUSE_DATABASE=test_seqr coverage run --source="./vlm" --omit="./vlm/__main__.py" -m pytest vlm/ coverage report --fail-under=95 diff --git a/vlm/liftover_references/setup_clickhouse_test_data.py b/vlm/liftover_references/setup_clickhouse_test_data.py new file mode 100644 index 0000000000..00b57c4eaa --- /dev/null +++ b/vlm/liftover_references/setup_clickhouse_test_data.py @@ -0,0 +1,31 @@ +import clickhouse_connect +import sys + +def setup_clickhouse_test_data(username, password): + client = clickhouse_connect.get_client(host='localhost', port=8123, username=username, password=password) + client.command('CREATE DATABASE test_seqr') + + client.command('CREATE TABLE seqr.`GRCh37/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId') + client.insert('GRCh37/SNV_INDEL/key_lookup', data=[['7-143270172-A-G', 1], ['1-39190091-T-G', 2]], database='test_seqr') + client.command( + 'CREATE DICTIONARY seqr.`GRCh37/SNV_INDEL/gt_stats_dict` (key UInt32, ac_wes UInt32, ac_wgs UInt32, hom_wes UInt32, hom_wgs UInt32) PRIMARY KEY key SOURCE(CLICKHOUSE(USER %s PASSWORD %s QUERY "SELECT * FROM VALUES ((1, 4104, 607, 1276, 232), (2, 7, 2, 2, 1))")) LIFETIME(0) LAYOUT(FLAT(MAX_ARRAY_SIZE 500000000))', + parameters=(username, password), + ) + + client.command('CREATE TABLE seqr.`GRCh38/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId') + client.insert('GRCh38/SNV_INDEL/key_lookup', data=[['1-38724419-T-G', 1]], database='test_seqr') + client.command( + 'CREATE DICTIONARY seqr.`GRCh38/SNV_INDEL/gt_stats_dict` (key UInt32, ac_wes UInt32, ac_wgs UInt32, hom_wes UInt32, hom_wgs UInt32) PRIMARY KEY key SOURCE(CLICKHOUSE(USER %s PASSWORD %s QUERY "SELECT * FROM VALUES ((1, 18, 10, 3, 1))")) LIFETIME(0) LAYOUT(FLAT(MAX_ARRAY_SIZE 500000000))', + parameters=(username, password), + ) + + client.command("CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'") + client.command('GRANT SELECT ON seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user') + client.command('GRANT SELECT ON seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user') + client.command('GRANT dictGet ON seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user') + client.command('GRANT dictGet ON seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user') + + +if __name__ == '__main__': + args = sys.argv[1:] + setup_clickhouse_test_data(*args) From b2a0b5aefe160e2fc9809cc28b931f8d2d8d7f4d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 11:56:26 -0400 Subject: [PATCH 06/46] move file to right location --- vlm/{liftover_references => }/setup_clickhouse_test_data.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename vlm/{liftover_references => }/setup_clickhouse_test_data.py (100%) diff --git a/vlm/liftover_references/setup_clickhouse_test_data.py b/vlm/setup_clickhouse_test_data.py similarity index 100% rename from vlm/liftover_references/setup_clickhouse_test_data.py rename to vlm/setup_clickhouse_test_data.py From 35c718ec1e1068f40902508148deafdc4f4fc820 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:04:42 -0400 Subject: [PATCH 07/46] fix test service port? --- .github/workflows/vlm-unit-tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index f44e97f801..f08cb793d9 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -49,7 +49,6 @@ jobs: clickhouse: image: bitnami/clickhouse:latest ports: - - 9000:9000 # Native client interface - 8123:8123 # HTTP interface options: >- --health-cmd "clickhouse-client --query 'SELECT 1'" From aaaadf43e3b285124fc958c904330cccc708bbaa Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:10:33 -0400 Subject: [PATCH 08/46] separate files for different backends --- .github/workflows/vlm-unit-tests.yaml | 4 +-- vlm/clickhouse_utils.py | 22 ++++++++++++++++ vlm/hail_backend_utils.py | 15 +++++++++++ vlm/match.py | 37 ++++----------------------- 4 files changed, 44 insertions(+), 34 deletions(-) create mode 100644 vlm/clickhouse_utils.py create mode 100644 vlm/hail_backend_utils.py diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index f08cb793d9..1bbed1e4b1 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -38,7 +38,7 @@ jobs: export NODE_ID=TestVLM export MACHINE_MEM=24 export JAVA_OPTS_XSS=16M - coverage run --source="./vlm" --omit="./vlm/__main__.py" -m pytest vlm/ + coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/clickhouse_utils.py" -m pytest vlm/ coverage report --fail-under=95 vlm_clickhouse: @@ -78,6 +78,6 @@ jobs: export CLICKHOUSE_USERNAME=vlm_test_user export CLICKHOUSE_PASSWORD=vlm_test_password export CLICKHOUSE_DATABASE=test_seqr - coverage run --source="./vlm" --omit="./vlm/__main__.py" -m pytest vlm/ + coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/hail_backend_utils.py" -m pytest vlm/ coverage report --fail-under=95 diff --git a/vlm/clickhouse_utils.py b/vlm/clickhouse_utils.py new file mode 100644 index 0000000000..654de39db6 --- /dev/null +++ b/vlm/clickhouse_utils.py @@ -0,0 +1,22 @@ +import clickhouse_connect +import hail as hl +import os + +CLICKHOUSE_CONNECTION_PARAMS = { + 'host': os.environ.get('CLICKHOUSE_SERVICE_HOSTNAME'), + 'port': os.environ.get('CLICKHOUSE_SERVICE_PORT'), + 'username': os.environ.get('CLICKHOUSE_USERNAME'), + 'password': os.environ.get('CLICKHOUSE_PASSWORD'), + 'database': os.environ.get('CLICKHOUSE_DATABASE', 'seqr'), +} + + +def get_clickhouse_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, genome_build: str) -> hl.Struct: + locus = hl.eval(locus) + client = clickhouse_connect.get_client(**CLICKHOUSE_CONNECTION_PARAMS) + results = client.query( + f"SELECT dictGet(`{genome_build}/SNV_INDEL/gt_stats_dict`, ('ac_wes', 'ac_wgs', 'hom_wes', 'hom_wgs'), key) FROM `{genome_build}/SNV_INDEL/key_lookup` WHERE variantId=%(variant_id)s", + parameters={'variant_id': f'{locus.contig.replace("chr", "")}-{locus.position}-{ref}-{alt}'}, + ).result_set + return (results[0]['ac_wes'] + results[0]['ac_wgs'], results[0]['hom_wes'] + results[0]['hom_wgs']) if results else (0, 0) + diff --git a/vlm/hail_backend_utils.py b/vlm/hail_backend_utils.py new file mode 100644 index 0000000000..6d12287870 --- /dev/null +++ b/vlm/hail_backend_utils.py @@ -0,0 +1,15 @@ +import hail as hl +import os + +VLM_DATA_DIR = os.environ.get('VLM_DATA_DIR') + + +def get_hail_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, genome_build: str) -> hl.Struct: + interval = hl.eval(hl.interval(locus, locus, includes_start=True, includes_end=True)) + ht = hl.read_table( + f'{VLM_DATA_DIR}/{genome_build}/SNV_INDEL/annotations.ht', _intervals=[interval], _filter_intervals=True, + ) + ht = ht.filter(ht.alleles == hl.array([ref, alt])) + + counts = ht.aggregate(hl.agg.take(ht.gt_stats, 1)) + return (counts[0].AC, counts[0].hom) if counts else (0, 0) diff --git a/vlm/match.py b/vlm/match.py index 6c15750465..7307940d0a 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -1,21 +1,15 @@ from aiohttp.web import HTTPBadRequest -import clickhouse_connect import hail as hl import os -VLM_DATA_DIR = os.environ.get('VLM_DATA_DIR') +from clickhouse_utils import get_clickhouse_variant_counts +from hail_backend_utils import get_hail_variant_counts + +CLICKHOUSE_ENABLED = bool(os.environ.get('CLICKHOUSE_SERVICE_HOSTNAME')) SEQR_BASE_URL = os.environ.get('SEQR_BASE_URL') VLM_DEFAULT_CONTACT_EMAIL = os.environ.get('VLM_DEFAULT_CONTACT_EMAIL') NODE_ID = os.environ.get('NODE_ID') -CLICKHOUSE_SERVICE_HOSTNAME = os.environ.get('CLICKHOUSE_SERVICE_HOSTNAME') -CLICKHOUSE_CONNECTION_PARAMS = { - 'host': CLICKHOUSE_SERVICE_HOSTNAME, - 'port': os.environ.get('CLICKHOUSE_SERVICE_PORT'), - 'username': os.environ.get('CLICKHOUSE_USERNAME'), - 'password': os.environ.get('CLICKHOUSE_PASSWORD'), - 'database': os.environ.get('CLICKHOUSE_DATABASE', 'seqr'), -} BEACON_HANDOVER_TYPE = { 'id': NODE_ID, @@ -48,7 +42,7 @@ def get_variant_match(query: dict) -> dict: chrom, pos, ref, alt, genome_build = _parse_match_query(query) locus = hl.locus(chrom, pos, reference_genome=genome_build) - get_counts_func = _get_clickhouse_variant_counts if CLICKHOUSE_SERVICE_HOSTNAME else _get_hail_variant_counts + get_counts_func = get_clickhouse_variant_counts if CLICKHOUSE_ENABLED else get_hail_variant_counts ac, hom = get_counts_func(locus, ref, alt, genome_build) liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 @@ -99,27 +93,6 @@ def _parse_match_query(query: dict) -> tuple[str, int, str, str, str]: return chrom, start, query['referenceBases'], query['alternateBases'], genome_build -def _get_hail_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, genome_build: str) -> hl.Struct: - interval = hl.eval(hl.interval(locus, locus, includes_start=True, includes_end=True)) - ht = hl.read_table( - f'{VLM_DATA_DIR}/{genome_build}/SNV_INDEL/annotations.ht', _intervals=[interval], _filter_intervals=True, - ) - ht = ht.filter(ht.alleles == hl.array([ref, alt])) - - counts = ht.aggregate(hl.agg.take(ht.gt_stats, 1)) - return (counts[0].AC, counts[0].hom) if counts else (0, 0) - - -def _get_clickhouse_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, genome_build: str) -> hl.Struct: - locus = hl.eval(locus) - client = clickhouse_connect.get_client(**CLICKHOUSE_CONNECTION_PARAMS) - results = client.query( - f"SELECT dictGet(`{genome_build}/SNV_INDEL/gt_stats_dict`, ('ac_wes', 'ac_wgs', 'hom_wes', 'hom_wgs'), key) FROM `{genome_build}/SNV_INDEL/key_lookup` WHERE variantId=%(variant_id)s", - parameters={'variant_id': f'{locus.contig.replace("chr", "")}-{locus.position}-{ref}-{alt}'}, - ).result_set - return (results[0]['ac_wes'] + results[0]['ac_wgs'], results[0]['hom_wes'] + results[0]['hom_wgs']) if results else (0, 0) - - def _format_results(ac: int, hom: int, url: str) -> dict: total = ac - hom # Homozygotes count twice toward the total AC result_sets = [ From cdfbc6e24c6c44172f6a82bffd8bca5026b801df Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:13:03 -0400 Subject: [PATCH 09/46] debug failed clickhouse setup --- vlm/setup_clickhouse_test_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vlm/setup_clickhouse_test_data.py b/vlm/setup_clickhouse_test_data.py index 00b57c4eaa..3f9a198029 100644 --- a/vlm/setup_clickhouse_test_data.py +++ b/vlm/setup_clickhouse_test_data.py @@ -2,7 +2,7 @@ import sys def setup_clickhouse_test_data(username, password): - client = clickhouse_connect.get_client(host='localhost', port=8123, username=username, password=password) + client = clickhouse_connect.get_client(host='localhost', port=8123, username=username, password=password, secure=True) client.command('CREATE DATABASE test_seqr') client.command('CREATE TABLE seqr.`GRCh37/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId') From 875e8d173a37dca822249f79521d11f4051a27e5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:14:53 -0400 Subject: [PATCH 10/46] fix imports --- vlm/match.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vlm/match.py b/vlm/match.py index 7307940d0a..0795e0f87b 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -2,8 +2,8 @@ import hail as hl import os -from clickhouse_utils import get_clickhouse_variant_counts -from hail_backend_utils import get_hail_variant_counts +from vlm.clickhouse_utils import get_clickhouse_variant_counts +from vlm.hail_backend_utils import get_hail_variant_counts CLICKHOUSE_ENABLED = bool(os.environ.get('CLICKHOUSE_SERVICE_HOSTNAME')) SEQR_BASE_URL = os.environ.get('SEQR_BASE_URL') From 80000111d827657b9edabe9057ee619de6fd6ff5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:16:17 -0400 Subject: [PATCH 11/46] better clickhosue health check --- .github/workflows/vlm-unit-tests.yaml | 2 +- vlm/setup_clickhouse_test_data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 1bbed1e4b1..4d6463b314 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -51,7 +51,7 @@ jobs: ports: - 8123:8123 # HTTP interface options: >- - --health-cmd "clickhouse-client --query 'SELECT 1'" + --health-cmd "curl http://localhost:8123" --health-interval 10s --health-timeout 5s --health-retries 5 diff --git a/vlm/setup_clickhouse_test_data.py b/vlm/setup_clickhouse_test_data.py index 3f9a198029..00b57c4eaa 100644 --- a/vlm/setup_clickhouse_test_data.py +++ b/vlm/setup_clickhouse_test_data.py @@ -2,7 +2,7 @@ import sys def setup_clickhouse_test_data(username, password): - client = clickhouse_connect.get_client(host='localhost', port=8123, username=username, password=password, secure=True) + client = clickhouse_connect.get_client(host='localhost', port=8123, username=username, password=password) client.command('CREATE DATABASE test_seqr') client.command('CREATE TABLE seqr.`GRCh37/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId') From ce6714f34196d22bf3ade377e8445914c24af64a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:22:07 -0400 Subject: [PATCH 12/46] codacy fix --- vlm/clickhouse_utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vlm/clickhouse_utils.py b/vlm/clickhouse_utils.py index 654de39db6..fb8bc14152 100644 --- a/vlm/clickhouse_utils.py +++ b/vlm/clickhouse_utils.py @@ -15,8 +15,11 @@ def get_clickhouse_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, locus = hl.eval(locus) client = clickhouse_connect.get_client(**CLICKHOUSE_CONNECTION_PARAMS) results = client.query( - f"SELECT dictGet(`{genome_build}/SNV_INDEL/gt_stats_dict`, ('ac_wes', 'ac_wgs', 'hom_wes', 'hom_wgs'), key) FROM `{genome_build}/SNV_INDEL/key_lookup` WHERE variantId=%(variant_id)s", - parameters={'variant_id': f'{locus.contig.replace("chr", "")}-{locus.position}-{ref}-{alt}'}, + "SELECT dictGet(%(dict_name)s, ('ac_wes', 'ac_wgs', 'hom_wes', 'hom_wgs'), key) FROM %(table_name)s WHERE variantId=%(variant_id)s", + parameters={ + 'variant_id': f'{locus.contig.replace("chr", "")}-{locus.position}-{ref}-{alt}', + 'table_name': f'{genome_build}/SNV_INDEL/key_lookup', + 'dict_name': f'{genome_build}/SNV_INDEL/gt_stats_dict', + }, ).result_set return (results[0]['ac_wes'] + results[0]['ac_wgs'], results[0]['hom_wes'] + results[0]['hom_wgs']) if results else (0, 0) - From 6afd9feeefa741f3ae36a65b5e8a9573f443851b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:22:54 -0400 Subject: [PATCH 13/46] fix coverage omit --- .github/workflows/vlm-unit-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 4d6463b314..36f84489cb 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -38,7 +38,7 @@ jobs: export NODE_ID=TestVLM export MACHINE_MEM=24 export JAVA_OPTS_XSS=16M - coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/clickhouse_utils.py" -m pytest vlm/ + coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data","./vlm/clickhouse_utils.py" -m pytest vlm/ coverage report --fail-under=95 vlm_clickhouse: @@ -78,6 +78,6 @@ jobs: export CLICKHOUSE_USERNAME=vlm_test_user export CLICKHOUSE_PASSWORD=vlm_test_password export CLICKHOUSE_DATABASE=test_seqr - coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/hail_backend_utils.py" -m pytest vlm/ + coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data","./vlm/hail_backend_utils.py" -m pytest vlm/ coverage report --fail-under=95 From aa087598eacbc3c356aa81f3d27be170e0bbd153 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:25:21 -0400 Subject: [PATCH 14/46] wrong clickhouse port? --- .github/workflows/vlm-unit-tests.yaml | 6 +++--- vlm/setup_clickhouse_test_data.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 36f84489cb..590d59e386 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -49,9 +49,9 @@ jobs: clickhouse: image: bitnami/clickhouse:latest ports: - - 8123:8123 # HTTP interface + - 9009:9009 # Inter-server HTTP interface options: >- - --health-cmd "curl http://localhost:8123" + --health-cmd "clickhouse-client --query 'SELECT 1'" --health-interval 10s --health-timeout 5s --health-retries 5 @@ -74,7 +74,7 @@ jobs: export SEQR_BASE_URL=https://test-seqr.org/ export NODE_ID=TestVLM export CLICKHOUSE_SERVICE_HOSTNAME=localhost - export CLICKHOUSE_SERVICE_PORT=8123 + export CLICKHOUSE_SERVICE_PORT=9009 export CLICKHOUSE_USERNAME=vlm_test_user export CLICKHOUSE_PASSWORD=vlm_test_password export CLICKHOUSE_DATABASE=test_seqr diff --git a/vlm/setup_clickhouse_test_data.py b/vlm/setup_clickhouse_test_data.py index 00b57c4eaa..30c113a335 100644 --- a/vlm/setup_clickhouse_test_data.py +++ b/vlm/setup_clickhouse_test_data.py @@ -2,7 +2,7 @@ import sys def setup_clickhouse_test_data(username, password): - client = clickhouse_connect.get_client(host='localhost', port=8123, username=username, password=password) + client = clickhouse_connect.get_client(host='localhost', port=9009, username=username, password=password) client.command('CREATE DATABASE test_seqr') client.command('CREATE TABLE seqr.`GRCh37/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId') From 5968467df72b32e080b4702ea0493349afe38e90 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:26:14 -0400 Subject: [PATCH 15/46] fix coverage omit for realz --- .github/workflows/vlm-unit-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 590d59e386..5dd973d811 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -38,7 +38,7 @@ jobs: export NODE_ID=TestVLM export MACHINE_MEM=24 export JAVA_OPTS_XSS=16M - coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data","./vlm/clickhouse_utils.py" -m pytest vlm/ + coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py","./vlm/clickhouse_utils.py" -m pytest vlm/ coverage report --fail-under=95 vlm_clickhouse: @@ -78,6 +78,6 @@ jobs: export CLICKHOUSE_USERNAME=vlm_test_user export CLICKHOUSE_PASSWORD=vlm_test_password export CLICKHOUSE_DATABASE=test_seqr - coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data","./vlm/hail_backend_utils.py" -m pytest vlm/ + coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py","./vlm/hail_backend_utils.py" -m pytest vlm/ coverage report --fail-under=95 From fe93663e590e251eb54812ae6f7e3a0ec6c48a32 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:28:58 -0400 Subject: [PATCH 16/46] debug --- .github/workflows/vlm-unit-tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 5dd973d811..f690b86e04 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -67,6 +67,8 @@ jobs: run: | python3 -m pip install --upgrade pip wheel pip install -r vlm/requirements-test.txt + - name: Debug clcikhouse + run: curl http://localhost:9009 - name: Set up Clickhouse Settings and Data run: python3 vlm/setup_clickhouse_test_data.py clickhouse_test_user clickhouse_test_password - name: Run coverage tests From 026932fd8794016adab48a48ff09b25a4da4180e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:32:27 -0400 Subject: [PATCH 17/46] debug --- .github/workflows/vlm-unit-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index f690b86e04..45c40ee152 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -49,7 +49,7 @@ jobs: clickhouse: image: bitnami/clickhouse:latest ports: - - 9009:9009 # Inter-server HTTP interface + - 8123:8123 # HTTP interface options: >- --health-cmd "clickhouse-client --query 'SELECT 1'" --health-interval 10s @@ -68,7 +68,7 @@ jobs: python3 -m pip install --upgrade pip wheel pip install -r vlm/requirements-test.txt - name: Debug clcikhouse - run: curl http://localhost:9009 + run: curl http://localhost:8123 - name: Set up Clickhouse Settings and Data run: python3 vlm/setup_clickhouse_test_data.py clickhouse_test_user clickhouse_test_password - name: Run coverage tests From fa0d195950e3db6846e05ff38b1fb97f8910ec54 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:33:32 -0400 Subject: [PATCH 18/46] debug --- .github/workflows/vlm-unit-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 45c40ee152..a507ae0751 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -49,7 +49,7 @@ jobs: clickhouse: image: bitnami/clickhouse:latest ports: - - 8123:8123 # HTTP interface + - 9000:9000 # HTTP interface options: >- --health-cmd "clickhouse-client --query 'SELECT 1'" --health-interval 10s @@ -68,7 +68,7 @@ jobs: python3 -m pip install --upgrade pip wheel pip install -r vlm/requirements-test.txt - name: Debug clcikhouse - run: curl http://localhost:8123 + run: curl http://localhost:9000 - name: Set up Clickhouse Settings and Data run: python3 vlm/setup_clickhouse_test_data.py clickhouse_test_user clickhouse_test_password - name: Run coverage tests From e1fc26542834fcd5c4b680145168d9b8808fc439 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:37:37 -0400 Subject: [PATCH 19/46] debug --- .github/workflows/vlm-unit-tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index a507ae0751..43a7c86e35 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -49,7 +49,7 @@ jobs: clickhouse: image: bitnami/clickhouse:latest ports: - - 9000:9000 # HTTP interface + - 8123:8123 # HTTP interface options: >- --health-cmd "clickhouse-client --query 'SELECT 1'" --health-interval 10s @@ -67,8 +67,8 @@ jobs: run: | python3 -m pip install --upgrade pip wheel pip install -r vlm/requirements-test.txt - - name: Debug clcikhouse - run: curl http://localhost:9000 + - name: Debug clickhouse + run: curl http://clickhouse:8123 - name: Set up Clickhouse Settings and Data run: python3 vlm/setup_clickhouse_test_data.py clickhouse_test_user clickhouse_test_password - name: Run coverage tests From 6c4c446761d508477c05ca568cd27b3f64de7380 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:41:05 -0400 Subject: [PATCH 20/46] use correct server and port for test data setup --- .github/workflows/vlm-unit-tests.yaml | 4 ++-- vlm/setup_clickhouse_test_data.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 43a7c86e35..b8160b4bae 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -70,12 +70,12 @@ jobs: - name: Debug clickhouse run: curl http://clickhouse:8123 - name: Set up Clickhouse Settings and Data - run: python3 vlm/setup_clickhouse_test_data.py clickhouse_test_user clickhouse_test_password + run: python3 vlm/setup_clickhouse_test_data.py clickhouse 8123 clickhouse_test_user clickhouse_test_password - name: Run coverage tests run: | export SEQR_BASE_URL=https://test-seqr.org/ export NODE_ID=TestVLM - export CLICKHOUSE_SERVICE_HOSTNAME=localhost + export CLICKHOUSE_SERVICE_HOSTNAME=clickhouse export CLICKHOUSE_SERVICE_PORT=9009 export CLICKHOUSE_USERNAME=vlm_test_user export CLICKHOUSE_PASSWORD=vlm_test_password diff --git a/vlm/setup_clickhouse_test_data.py b/vlm/setup_clickhouse_test_data.py index 30c113a335..aa1c6a6e8a 100644 --- a/vlm/setup_clickhouse_test_data.py +++ b/vlm/setup_clickhouse_test_data.py @@ -1,8 +1,8 @@ import clickhouse_connect import sys -def setup_clickhouse_test_data(username, password): - client = clickhouse_connect.get_client(host='localhost', port=9009, username=username, password=password) +def setup_clickhouse_test_data(host, port, username, password): + client = clickhouse_connect.get_client(host=host, port=port, username=username, password=password) client.command('CREATE DATABASE test_seqr') client.command('CREATE TABLE seqr.`GRCh37/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId') From da5ae7f505ddf0210db97dfe176b418f6494d62b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:44:17 -0400 Subject: [PATCH 21/46] fix db name in data setup --- vlm/setup_clickhouse_test_data.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/vlm/setup_clickhouse_test_data.py b/vlm/setup_clickhouse_test_data.py index aa1c6a6e8a..a50334040c 100644 --- a/vlm/setup_clickhouse_test_data.py +++ b/vlm/setup_clickhouse_test_data.py @@ -5,25 +5,25 @@ def setup_clickhouse_test_data(host, port, username, password): client = clickhouse_connect.get_client(host=host, port=port, username=username, password=password) client.command('CREATE DATABASE test_seqr') - client.command('CREATE TABLE seqr.`GRCh37/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId') + client.command('CREATE TABLE test_seqr.`GRCh37/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId') client.insert('GRCh37/SNV_INDEL/key_lookup', data=[['7-143270172-A-G', 1], ['1-39190091-T-G', 2]], database='test_seqr') client.command( - 'CREATE DICTIONARY seqr.`GRCh37/SNV_INDEL/gt_stats_dict` (key UInt32, ac_wes UInt32, ac_wgs UInt32, hom_wes UInt32, hom_wgs UInt32) PRIMARY KEY key SOURCE(CLICKHOUSE(USER %s PASSWORD %s QUERY "SELECT * FROM VALUES ((1, 4104, 607, 1276, 232), (2, 7, 2, 2, 1))")) LIFETIME(0) LAYOUT(FLAT(MAX_ARRAY_SIZE 500000000))', + 'CREATE DICTIONARY test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` (key UInt32, ac_wes UInt32, ac_wgs UInt32, hom_wes UInt32, hom_wgs UInt32) PRIMARY KEY key SOURCE(CLICKHOUSE(USER %s PASSWORD %s QUERY "SELECT * FROM VALUES ((1, 4104, 607, 1276, 232), (2, 7, 2, 2, 1))")) LIFETIME(0) LAYOUT(FLAT(MAX_ARRAY_SIZE 500000000))', parameters=(username, password), ) - client.command('CREATE TABLE seqr.`GRCh38/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId') + client.command('CREATE TABLE test_seqr.`GRCh38/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId') client.insert('GRCh38/SNV_INDEL/key_lookup', data=[['1-38724419-T-G', 1]], database='test_seqr') client.command( - 'CREATE DICTIONARY seqr.`GRCh38/SNV_INDEL/gt_stats_dict` (key UInt32, ac_wes UInt32, ac_wgs UInt32, hom_wes UInt32, hom_wgs UInt32) PRIMARY KEY key SOURCE(CLICKHOUSE(USER %s PASSWORD %s QUERY "SELECT * FROM VALUES ((1, 18, 10, 3, 1))")) LIFETIME(0) LAYOUT(FLAT(MAX_ARRAY_SIZE 500000000))', + 'CREATE DICTIONARY test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` (key UInt32, ac_wes UInt32, ac_wgs UInt32, hom_wes UInt32, hom_wgs UInt32) PRIMARY KEY key SOURCE(CLICKHOUSE(USER %s PASSWORD %s QUERY "SELECT * FROM VALUES ((1, 18, 10, 3, 1))")) LIFETIME(0) LAYOUT(FLAT(MAX_ARRAY_SIZE 500000000))', parameters=(username, password), ) client.command("CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'") - client.command('GRANT SELECT ON seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user') - client.command('GRANT SELECT ON seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user') - client.command('GRANT dictGet ON seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user') - client.command('GRANT dictGet ON seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user') + client.command('GRANT SELECT ON test_seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user') + client.command('GRANT SELECT ON test_seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user') + client.command('GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user') + client.command('GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user') if __name__ == '__main__': From 13ba564bc53b96341446168322715a9eef27a895 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:47:32 -0400 Subject: [PATCH 22/46] fix port --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index b8160b4bae..7f91202a6c 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -76,7 +76,7 @@ jobs: export SEQR_BASE_URL=https://test-seqr.org/ export NODE_ID=TestVLM export CLICKHOUSE_SERVICE_HOSTNAME=clickhouse - export CLICKHOUSE_SERVICE_PORT=9009 + export CLICKHOUSE_SERVICE_PORT=8123 export CLICKHOUSE_USERNAME=vlm_test_user export CLICKHOUSE_PASSWORD=vlm_test_password export CLICKHOUSE_DATABASE=test_seqr From 5322c536e2481f50752e530ecd865de2c43a6d57 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 12:57:53 -0400 Subject: [PATCH 23/46] better query syntax --- vlm/clickhouse_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vlm/clickhouse_utils.py b/vlm/clickhouse_utils.py index fb8bc14152..6d5d0553be 100644 --- a/vlm/clickhouse_utils.py +++ b/vlm/clickhouse_utils.py @@ -15,11 +15,11 @@ def get_clickhouse_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, locus = hl.eval(locus) client = clickhouse_connect.get_client(**CLICKHOUSE_CONNECTION_PARAMS) results = client.query( - "SELECT dictGet(%(dict_name)s, ('ac_wes', 'ac_wgs', 'hom_wes', 'hom_wgs'), key) FROM %(table_name)s WHERE variantId=%(variant_id)s", + "SELECT plus(gt_stats.1, gt_stats.2), plus(gt_stats.3, gt_stats.4) FROM (SELECT dictGet(%(dict_name)s, ('ac_wes', 'ac_wgs', 'hom_wes', 'hom_wgs'), key) AS gt_stats FROM %(table_name)s WHERE variantId=%(variant_id)s)", parameters={ 'variant_id': f'{locus.contig.replace("chr", "")}-{locus.position}-{ref}-{alt}', 'table_name': f'{genome_build}/SNV_INDEL/key_lookup', 'dict_name': f'{genome_build}/SNV_INDEL/gt_stats_dict', }, ).result_set - return (results[0]['ac_wes'] + results[0]['ac_wgs'], results[0]['hom_wes'] + results[0]['hom_wgs']) if results else (0, 0) + return results[0] if results else (0, 0) From c65d0af564defc117a2272fc554bb3349ab4de09 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 13:04:29 -0400 Subject: [PATCH 24/46] remove debug --- .github/workflows/vlm-unit-tests.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 7f91202a6c..56774ef679 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -67,8 +67,6 @@ jobs: run: | python3 -m pip install --upgrade pip wheel pip install -r vlm/requirements-test.txt - - name: Debug clickhouse - run: curl http://clickhouse:8123 - name: Set up Clickhouse Settings and Data run: python3 vlm/setup_clickhouse_test_data.py clickhouse 8123 clickhouse_test_user clickhouse_test_password - name: Run coverage tests From 491e145e3ff44c9e5c33caf4fa7889b34d0dae61 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 16:27:21 -0400 Subject: [PATCH 25/46] ui tweak --- ui/pages/DataManagement/components/TriggerDagPages.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/pages/DataManagement/components/TriggerDagPages.jsx b/ui/pages/DataManagement/components/TriggerDagPages.jsx index db1b456fd2..bdd0b28212 100644 --- a/ui/pages/DataManagement/components/TriggerDagPages.jsx +++ b/ui/pages/DataManagement/components/TriggerDagPages.jsx @@ -52,7 +52,7 @@ const REFERENCE_DATASET_FIELDS = [ const TriggerDagForm = ({ dagName, fields }) => ( From 23c3e4c9c44a0aecb809c6e78d0cb5bbbc25afbe Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 16:35:50 -0400 Subject: [PATCH 26/46] remove references t dag from UI --- ui/pages/DataManagement/DataManagement.jsx | 6 ++-- ...s.jsx => TriggerSearchDataUpdatePages.jsx} | 29 ++++++++++--------- 2 files changed, 18 insertions(+), 17 deletions(-) rename ui/pages/DataManagement/components/{TriggerDagPages.jsx => TriggerSearchDataUpdatePages.jsx} (65%) diff --git a/ui/pages/DataManagement/DataManagement.jsx b/ui/pages/DataManagement/DataManagement.jsx index fd63d3bd2e..db2e0db7de 100644 --- a/ui/pages/DataManagement/DataManagement.jsx +++ b/ui/pages/DataManagement/DataManagement.jsx @@ -12,7 +12,7 @@ import LoadData from './components/LoadData' import RnaSeq from './components/RnaSeq' import Users from './components/Users' import PhenotypePrioritization from './components/PhenotypePrioritization' -import TRIGGER_DAG_PAGES from './components/TriggerDagPages' +import TRIGGER_SEARCH_DATA_UPDATE_PAGES from './components/TriggerSearchDataUpdatePages' const IFRAME_STYLE = { position: 'fixed', left: '0', top: '95px' } @@ -51,7 +51,7 @@ const LOCAL_HAIL_SEARCH_DATA_MANAGEMENT_PAGES = [ const AIRFLOW_HAIL_SEARCH_DATA_MANAGEMENT_PAGES = [ ...DATA_MANAGEMENT_PAGES, - ...TRIGGER_DAG_PAGES, + ...TRIGGER_SEARCH_DATA_UPDATE_PAGES, ] const dataManagementPages = (user, elasticsearchEnabled) => { @@ -61,7 +61,7 @@ const dataManagementPages = (user, elasticsearchEnabled) => { if (elasticsearchEnabled) { return ES_DATA_MANAGEMENT_PAGES } - return user.isAnvil ? AIRFLOW_HAIL_SEARCH_DATA_MANAGEMENT_PAGES : LOCAL_HAIL_SEARCH_DATA_MANAGEMENT_PAGES + return !user.isAnvil ? AIRFLOW_HAIL_SEARCH_DATA_MANAGEMENT_PAGES : LOCAL_HAIL_SEARCH_DATA_MANAGEMENT_PAGES } const DataManagement = ({ match, user, pages }) => ( diff --git a/ui/pages/DataManagement/components/TriggerDagPages.jsx b/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx similarity index 65% rename from ui/pages/DataManagement/components/TriggerDagPages.jsx rename to ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx index bdd0b28212..5041db50ef 100644 --- a/ui/pages/DataManagement/components/TriggerDagPages.jsx +++ b/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx @@ -11,6 +11,7 @@ import { DATASET_TYPE_MITO_CALLS, GENOME_VERSION_FIELD, } from 'shared/utils/constants' +import { snakecaseToTitlecase } from 'shared/utils/stringUtils' const DATASET_TYPE_FIELD = { name: 'datasetType', @@ -50,33 +51,33 @@ const REFERENCE_DATASET_FIELDS = [ DATASET_TYPE_FIELD, ] -const TriggerDagForm = ({ dagName, fields }) => ( +const TriggerSearchDataUpdateForm = ({ path, fields }) => ( ) -TriggerDagForm.propTypes = { - dagName: PropTypes.string, +TriggerSearchDataUpdateForm.propTypes = { + path: PropTypes.string, fields: PropTypes.arrayOf(PropTypes.object), } -const TriggerDeleteProjectsDag = () => ( - +const TriggerDeleteProjects = () => ( + ) -const TriggerDeleteFamiliesDag = () => ( - +const TriggerDeleteFamilies = () => ( + ) -const TriggerUpdateReferenceDatasetDag = () => ( - +const TriggerUpdateReferenceDataset = () => ( + ) export default [ - { path: 'delete_search_projects', component: TriggerDeleteProjectsDag }, - { path: 'delete_search_families', component: TriggerDeleteFamiliesDag }, - { path: 'update_search_reference_data', component: TriggerUpdateReferenceDatasetDag }, + { path: 'delete_search_project', component: TriggerDeleteProjects }, + { path: 'delete_search_family', component: TriggerDeleteFamilies }, + { path: 'update_search_reference_data', component: TriggerUpdateReferenceDataset }, ] From b22ed1cd0d70f7e6800769a2f4d32c5dc320231f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 16:41:52 -0400 Subject: [PATCH 27/46] update URLS --- seqr/urls.py | 7 ++- seqr/views/apis/data_manager_api.py | 44 ++++++++++++++++++- .../TriggerSearchDataUpdatePages.jsx | 2 +- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/seqr/urls.py b/seqr/urls.py index 07888d2b95..f93c3a603a 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -124,7 +124,8 @@ from seqr.views.apis.data_manager_api import elasticsearch_status, delete_index, \ update_rna_seq, load_rna_seq_sample_data, proxy_to_kibana, load_phenotype_prioritization_data, \ - validate_callset, get_loaded_projects, load_data, loading_vcfs, trigger_dag, proxy_to_luigi + validate_callset, get_loaded_projects, load_data, loading_vcfs, proxy_to_luigi, \ + trigger_delete_project, trigger_delete_family, trigger_update_search_reference_data from seqr.views.apis.report_api import \ anvil_export, \ family_metadata, \ @@ -339,7 +340,9 @@ 'data_management/loaded_projects/(?P[^/]+)/(?P[^/]+)/(?P[^/]+)': get_loaded_projects, 'data_management/load_data': load_data, 'data_management/add_igv': receive_bulk_igv_table_handler, - 'data_management/trigger_dag/(?P[^/]+)': trigger_dag, + 'data_management/trigger_delete_project': trigger_delete_project, + 'data_management/trigger_delete_family': trigger_delete_family, + 'data_management/trigger_update_search_reference_data': trigger_update_search_reference_data, 'summary_data/saved_variants/(?P[^/]+)': saved_variants_page, 'summary_data/hpo/(?P[^/]+)': hpo_summary_data, diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 65c8d599e7..e1ab672f34 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -425,7 +425,49 @@ def fetch_missing_vcf_samples(missing_vcf_samples): @data_manager_required -def trigger_dag(request, dag_id): +def trigger_delete_project(request): + if not is_airflow_enabled(): + raise PermissionDenied() + request_json = json.loads(request.body) + project_guid = request_json.pop('project', None) + family_guid = request_json.pop('family', None) + kwargs = {_to_snake_case(k): v for k, v in request_json.items()} + project = None + if project_guid: + project = Project.objects.get(guid=project_guid) + elif family_guid: + project = Project.objects.get(family__guid=family_guid) + kwargs['family_guids'] = [family_guid] + try: + dag_variables = trigger_airflow_dag(dag_id, project, **kwargs) + except Exception as e: + return create_json_response({'error': str(e)}, status=400) + return create_json_response({'info': [f'Triggered DAG {dag_id} with variables: {json.dumps(dag_variables)}']}) + + +@data_manager_required +def trigger_delete_family(request): + if not is_airflow_enabled(): + raise PermissionDenied() + request_json = json.loads(request.body) + project_guid = request_json.pop('project', None) + family_guid = request_json.pop('family', None) + kwargs = {_to_snake_case(k): v for k, v in request_json.items()} + project = None + if project_guid: + project = Project.objects.get(guid=project_guid) + elif family_guid: + project = Project.objects.get(family__guid=family_guid) + kwargs['family_guids'] = [family_guid] + try: + dag_variables = trigger_airflow_dag(dag_id, project, **kwargs) + except Exception as e: + return create_json_response({'error': str(e)}, status=400) + return create_json_response({'info': [f'Triggered DAG {dag_id} with variables: {json.dumps(dag_variables)}']}) + + +@data_manager_required +def trigger_update_search_reference_data(request): if not is_airflow_enabled(): raise PermissionDenied() request_json = json.loads(request.body) diff --git a/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx b/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx index 5041db50ef..7aa4ca250b 100644 --- a/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx +++ b/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx @@ -54,7 +54,7 @@ const REFERENCE_DATASET_FIELDS = [ const TriggerSearchDataUpdateForm = ({ path, fields }) => ( ) From be98e9484b7595ff50439a9ec73d11d2f75a39a9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 17:00:25 -0400 Subject: [PATCH 28/46] backend specific trgger functiosn --- clickhouse_search/search.py | 4 ++ seqr/views/apis/data_manager_api.py | 80 +++++++++++------------------ 2 files changed, 33 insertions(+), 51 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 31ed7fba11..726af52723 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -531,3 +531,7 @@ def get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids, revers batch = variants_ids[i:i + BATCH_SIZE] lookup.update(dict(key_lookup_class.objects.filter(variant_id__in=batch).values_list(*fields))) return lookup + + +def trigger_delete_clickhouse_project(project=None, dataset_type=None, **kwargs): + return 'Oops!' diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index e1ab672f34..67ff8c8b39 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -4,20 +4,19 @@ import gzip import json import os -import re import requests import urllib3 -from django.contrib.postgres.aggregates import ArrayAgg from django.core.exceptions import PermissionDenied from django.db.models import Max, F, Q from django.http.response import HttpResponse from django.views.decorators.csrf import csrf_exempt from requests.exceptions import ConnectionError as RequestConnectionError +from clickhouse_search.search import trigger_delete_clickhouse_project from seqr.utils.communication_utils import send_project_notification from seqr.utils.search.add_data_utils import prepare_data_loading_request, get_loading_samples_validator -from seqr.utils.search.utils import get_search_backend_status, delete_search_backend_data +from seqr.utils.search.utils import get_search_backend_status, delete_search_backend_data, backend_specific_call from seqr.utils.file_utils import file_iter, does_file_exist from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException @@ -27,7 +26,7 @@ from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS from seqr.views.utils.dataset_utils import load_rna_seq, load_phenotype_prioritization_data_file, RNA_DATA_TYPE_CONFIGS, \ post_process_rna_data, convert_django_meta_to_http_headers -from seqr.views.utils.file_utils import parse_file, get_temp_file_path, load_uploaded_file, persist_temp_file +from seqr.views.utils.file_utils import get_temp_file_path, load_uploaded_file, persist_temp_file from seqr.views.utils.json_utils import create_json_response, _to_snake_case from seqr.views.utils.json_to_orm_utils import update_model_from_json from seqr.views.utils.pedigree_info_utils import get_validated_related_individuals, JsonConstants @@ -426,65 +425,44 @@ def fetch_missing_vcf_samples(missing_vcf_samples): @data_manager_required def trigger_delete_project(request): - if not is_airflow_enabled(): - raise PermissionDenied() - request_json = json.loads(request.body) - project_guid = request_json.pop('project', None) - family_guid = request_json.pop('family', None) - kwargs = {_to_snake_case(k): v for k, v in request_json.items()} - project = None - if project_guid: - project = Project.objects.get(guid=project_guid) - elif family_guid: - project = Project.objects.get(family__guid=family_guid) - kwargs['family_guids'] = [family_guid] - try: - dag_variables = trigger_airflow_dag(dag_id, project, **kwargs) - except Exception as e: - return create_json_response({'error': str(e)}, status=400) - return create_json_response({'info': [f'Triggered DAG {dag_id} with variables: {json.dumps(dag_variables)}']}) + return _trigger_data_update(request, 'DELETE_PROJECTS', clickhouse_func=trigger_delete_clickhouse_project) @data_manager_required def trigger_delete_family(request): - if not is_airflow_enabled(): - raise PermissionDenied() - request_json = json.loads(request.body) - project_guid = request_json.pop('project', None) - family_guid = request_json.pop('family', None) - kwargs = {_to_snake_case(k): v for k, v in request_json.items()} - project = None - if project_guid: - project = Project.objects.get(guid=project_guid) - elif family_guid: - project = Project.objects.get(family__guid=family_guid) - kwargs['family_guids'] = [family_guid] - try: - dag_variables = trigger_airflow_dag(dag_id, project, **kwargs) - except Exception as e: - return create_json_response({'error': str(e)}, status=400) - return create_json_response({'info': [f'Triggered DAG {dag_id} with variables: {json.dumps(dag_variables)}']}) + return _trigger_data_update(request, 'DELETE_FAMILIES') @data_manager_required def trigger_update_search_reference_data(request): - if not is_airflow_enabled(): - raise PermissionDenied() + return _trigger_data_update(request, 'UPDATE_REFERENCE_DATASETS') + + +def _raise_backend_not_implemented(*args, **kwargs): + raise ErrorsWarningsException(['This functionality is not available in the current search backend']) + + +def _trigger_data_update(request, dag_id, clickhouse_func=_raise_backend_not_implemented): request_json = json.loads(request.body) - project_guid = request_json.pop('project', None) - family_guid = request_json.pop('family', None) kwargs = {_to_snake_case(k): v for k, v in request_json.items()} - project = None - if project_guid: - project = Project.objects.get(guid=project_guid) - elif family_guid: - project = Project.objects.get(family__guid=family_guid) - kwargs['family_guids'] = [family_guid] + info = backend_specific_call(_raise_backend_not_implemented, _trigger_dag, clickhouse_func)(dag_id=dag_id, **kwargs) + return create_json_response({'info': [info]}) + + +def _trigger_dag(dag_id, project=None, family=None, **kwargs): + if not is_airflow_enabled(): + raise PermissionDenied() + project_model = None + if project: + project_model = Project.objects.get(guid=project) + elif family: + project_model = Project.objects.get(family__guid=family) + kwargs['family_guids'] = [family] try: - dag_variables = trigger_airflow_dag(dag_id, project, **kwargs) + dag_variables = trigger_airflow_dag(dag_id, project_model, **kwargs) except Exception as e: - return create_json_response({'error': str(e)}, status=400) - return create_json_response({'info': [f'Triggered DAG {dag_id} with variables: {json.dumps(dag_variables)}']}) + raise ErrorsWarningsException([str(e)]) + return f'Triggered DAG {dag_id} with variables: {json.dumps(dag_variables)}' # Hop-by-hop HTTP response headers shouldn't be forwarded. From 85e021004026a2cf5159ff8795858685ff447c36 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 Aug 2025 10:37:51 -0400 Subject: [PATCH 29/46] shared project models getting --- clickhouse_search/search.py | 2 +- seqr/views/apis/data_manager_api.py | 34 ++++++++++++++++------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 726af52723..a3bb985803 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -533,5 +533,5 @@ def get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids, revers return lookup -def trigger_delete_clickhouse_project(project=None, dataset_type=None, **kwargs): +def trigger_delete_clickhouse_project(project, dataset_type=None, **kwargs): return 'Oops!' diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 67ff8c8b39..af0f460c7e 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -425,41 +425,45 @@ def fetch_missing_vcf_samples(missing_vcf_samples): @data_manager_required def trigger_delete_project(request): - return _trigger_data_update(request, 'DELETE_PROJECTS', clickhouse_func=trigger_delete_clickhouse_project) + request_json = json.loads(request.body) + project_guid = request_json.pop('project') + project = Project.objects.get(guid=project_guid) + return _trigger_data_update( + project, dag_id='DELETE_PROJECTS', clickhouse_func=trigger_delete_clickhouse_project, **request_json, + ) @data_manager_required def trigger_delete_family(request): - return _trigger_data_update(request, 'DELETE_FAMILIES') + request_json = json.loads(request.body) + family_guid = request_json.pop('family') + project = Project.objects.get(family__guid=family_guid) + return _trigger_data_update(project, family_guid=family_guid, dag_id='DELETE_FAMILIES', **request_json) @data_manager_required def trigger_update_search_reference_data(request): - return _trigger_data_update(request, 'UPDATE_REFERENCE_DATASETS') + request_json = json.loads(request.body) + return _trigger_data_update(project=None, dag_id='UPDATE_REFERENCE_DATASETS', **request_json) def _raise_backend_not_implemented(*args, **kwargs): raise ErrorsWarningsException(['This functionality is not available in the current search backend']) -def _trigger_data_update(request, dag_id, clickhouse_func=_raise_backend_not_implemented): - request_json = json.loads(request.body) - kwargs = {_to_snake_case(k): v for k, v in request_json.items()} - info = backend_specific_call(_raise_backend_not_implemented, _trigger_dag, clickhouse_func)(dag_id=dag_id, **kwargs) +def _trigger_data_update(project, clickhouse_func=_raise_backend_not_implemented, **kwargs): + kwargs = {_to_snake_case(k): v for k, v in kwargs.items()} + info = backend_specific_call(_raise_backend_not_implemented, _trigger_dag, clickhouse_func)(project, **kwargs) return create_json_response({'info': [info]}) -def _trigger_dag(dag_id, project=None, family=None, **kwargs): +def _trigger_dag(project, dag_id=None, family_guid=None, **kwargs): if not is_airflow_enabled(): raise PermissionDenied() - project_model = None - if project: - project_model = Project.objects.get(guid=project) - elif family: - project_model = Project.objects.get(family__guid=family) - kwargs['family_guids'] = [family] + if family_guid: + kwargs['family_guids'] = [family_guid] try: - dag_variables = trigger_airflow_dag(dag_id, project_model, **kwargs) + dag_variables = trigger_airflow_dag(dag_id, project, **kwargs) except Exception as e: raise ErrorsWarningsException([str(e)]) return f'Triggered DAG {dag_id} with variables: {json.dumps(dag_variables)}' From 43e2152ba50f5945e387f88cc0dea4597edbf88d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 Aug 2025 10:57:03 -0400 Subject: [PATCH 30/46] actually delete project data --- clickhouse_search/search.py | 14 +++++++++++--- seqr/views/apis/data_manager_api.py | 4 ++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index a3bb985803..3216235123 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -2,6 +2,7 @@ from collections import defaultdict from django.contrib.postgres.aggregates import ArrayAgg from django.core.exceptions import ObjectDoesNotExist +from django.db import connections from django.db.models import F, Min, Q from django.db.models.functions import JSONObject import json @@ -11,7 +12,7 @@ ArrayDistinct, ArrayMap from clickhouse_search.models import ENTRY_CLASS_MAP, ANNOTATIONS_CLASS_MAP, TRANSCRIPTS_CLASS_MAP, KEY_LOOKUP_CLASS_MAP, \ BaseClinvar, BaseAnnotationsMitoSnvIndel, BaseAnnotationsGRCh37SnvIndel, BaseAnnotationsSvGcnv -from reference_data.models import GeneConstraint, Omim +from reference_data.models import GeneConstraint, Omim, GENOME_VERSION_LOOKUP from seqr.models import Sample, PhenotypePrioritization from seqr.utils.logging_utils import SeqrLogger from seqr.utils.search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \ @@ -533,5 +534,12 @@ def get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids, revers return lookup -def trigger_delete_clickhouse_project(project, dataset_type=None, **kwargs): - return 'Oops!' +def delete_clickhouse_project(project, dataset_type=None, **kwargs): + table_base = f'{GENOME_VERSION_LOOKUP[project.genome_version]}/{dataset_type}' + with connections['clickhouse_write'].cursor() as cursor: + cursor.execute('ALTER TABLE %s DROP PARTITION %s', [f'{table_base}/entries', project.guid]) + view_name = f'{table_base}/project_gt_stats_to_gt_stats_mv' + cursor.execute('SYSTEM REFRESH VIEW %s', [view_name]) + cursor.execute('SYSTEM WAIT VIEW %s', [view_name]) + cursor.execute('SYSTEM RELOAD DICTIONARY %s', [f'{table_base}/gt_stats_dict']) + return f'Deleted all {dataset_type} search data for project {project.name}' diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index af0f460c7e..5992cd9e2e 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -13,7 +13,7 @@ from django.views.decorators.csrf import csrf_exempt from requests.exceptions import ConnectionError as RequestConnectionError -from clickhouse_search.search import trigger_delete_clickhouse_project +from clickhouse_search.search import delete_clickhouse_project from seqr.utils.communication_utils import send_project_notification from seqr.utils.search.add_data_utils import prepare_data_loading_request, get_loading_samples_validator from seqr.utils.search.utils import get_search_backend_status, delete_search_backend_data, backend_specific_call @@ -429,7 +429,7 @@ def trigger_delete_project(request): project_guid = request_json.pop('project') project = Project.objects.get(guid=project_guid) return _trigger_data_update( - project, dag_id='DELETE_PROJECTS', clickhouse_func=trigger_delete_clickhouse_project, **request_json, + project, dag_id='DELETE_PROJECTS', clickhouse_func=delete_clickhouse_project, **request_json, ) From 8fc801ef776a0489e1c152e562577aaf52a8fb9c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 Aug 2025 11:29:19 -0400 Subject: [PATCH 31/46] data manager tests for all 3 backends --- seqr/views/apis/data_manager_api_tests.py | 75 +++++++++++++++-------- seqr/views/utils/test_utils.py | 6 +- 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 5021eadf80..6e983d1895 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -9,7 +9,7 @@ from seqr.utils.communication_utils import _set_bulk_notification_stream from seqr.views.apis.data_manager_api import elasticsearch_status, delete_index, \ update_rna_seq, load_rna_seq_sample_data, load_phenotype_prioritization_data, validate_callset, loading_vcfs, \ - get_loaded_projects, trigger_dag, load_data + get_loaded_projects, load_data, trigger_delete_project, trigger_delete_family, trigger_update_search_reference_data from seqr.views.utils.orm_to_json_utils import _get_json_for_models from seqr.views.utils.test_utils import AuthenticationTestCase, AirflowTestCase, AirtableTest from seqr.utils.search.elasticsearch.es_utils_tests import urllib3_responses @@ -1515,10 +1515,12 @@ def _test_no_affected_family(self, url, body): Individual.objects.filter(guid='I000009_na20874').update(affected='A') @responses.activate - def test_trigger_dag(self): - self.check_data_manager_login(reverse(trigger_dag, args=['some_dag'])) + def test_trigger_delete_project(self): + url = reverse(trigger_delete_project) + self.check_data_manager_login(url) self._test_trigger_single_dag( + url, 'DELETE_PROJECTS', {'project': PROJECT_GUID, 'datasetType': 'SNV_INDEL'}, { @@ -1527,7 +1529,14 @@ def test_trigger_dag(self): 'reference_genome': 'GRCh37', } ) + + @responses.activate + def test_trigger_delete_family(self): + url = reverse(trigger_delete_family) + self.check_data_manager_login(url) + self._test_trigger_single_dag( + url, 'DELETE_FAMILIES', {'family': 'F000012_12', 'datasetType': 'MITO'}, { @@ -1538,29 +1547,26 @@ def test_trigger_dag(self): } ) + @responses.activate + def test_trigger_update_search_reference_data(self): + url = reverse(trigger_update_search_reference_data) + self.check_data_manager_login(url) + body = {'genomeVersion': '38', 'datasetType': 'SV'} - url = self._test_trigger_single_dag('UPDATE_REFERENCE_DATASETS',body,{ + self._test_trigger_single_dag(url, 'UPDATE_REFERENCE_DATASETS', body, { 'projects_to_run': None, 'dataset_type': 'SV', 'reference_genome': 'GRCh38', }) - self._test_dag_trigger_errors(url, body) - - def _test_trigger_single_dag(self, dag_id, body, dag_variables): - responses.calls.reset() - self.set_up_one_dag(dag_id, variables=dag_variables) - - url = reverse(trigger_dag, args=[dag_id]) + def _test_trigger_single_dag(self, url, dag_id, body, dag_variables): response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self._assert_expected_dag_trigger(response, dag_id, dag_variables) - return url def _assert_expected_dag_trigger(self, response, dag_id, variables): - self.assertEqual(response.status_code, 403) + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), {'errors': ['This functionality is not available in the current search backend'], 'warnings': None}) - def _test_dag_trigger_errors(self, url, body): - pass class LocalDataManagerAPITest(AuthenticationTestCase, DataManagerAPITest): fixtures = ['users', '1kg_project', 'reference_data'] @@ -2009,6 +2015,23 @@ def _add_update_check_dag_responses(self, variables=None, **kwargs): return super()._add_update_check_dag_responses(**kwargs) + def _assert_expected_airtable_errors(self, url): + responses.replace( + responses.GET, 'https://api.airtable.com/v0/app3Y97xtbbaOopVR/Samples', + json=INVALID_AIRTABLE_SAMPLE_RECORDS, status=200, + ) + response = self.client.get(url) + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), { + 'error': 'The following samples are associated with misconfigured PDOs in Airtable: HG00731, NA21234', + }) + +@mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') +class HailBackendDataManagerAPITest(AnvilDataManagerAPITest): + fixtures = ['users', 'social_auth', '1kg_project', 'reference_data'] + + CLICKHOUSE_HOSTNAME = '' + def _assert_update_check_airflow_calls(self, call_count, offset, update_check_path): if self.DAG_NAME != 'LOADING_PIPELINE': update_check_path = f'{self.MOCK_AIRFLOW_URL}/api/v1/variables/{self.DAG_NAME}' @@ -2026,20 +2049,20 @@ def _assert_expected_dag_trigger(self, response, dag_id, variables): self.assert_airflow_calls(variables, 5) - def _test_dag_trigger_errors(self, url, body): + def _test_trigger_single_dag(self, url, dag_id, body, dag_variables): + responses.calls.reset() + self.set_up_one_dag(dag_id, variables=dag_variables) + + super()._test_trigger_single_dag(url, dag_id, body, dag_variables) + self.set_dag_trigger_error_response() response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'error': 'UPDATE_REFERENCE_DATASETS DAG is running and cannot be triggered again.'}) - - def _assert_expected_airtable_errors(self, url): - responses.replace( - responses.GET, 'https://api.airtable.com/v0/app3Y97xtbbaOopVR/Samples', - json=INVALID_AIRTABLE_SAMPLE_RECORDS, status=200, - ) - response = self.client.get(url) - self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), { - 'error': 'The following samples are associated with misconfigured PDOs in Airtable: HG00731, NA21234', + 'errors': [f'{dag_id} DAG is running and cannot be triggered again.'], 'warnings': None, }) + # DAG trigger not attempted when airflow is disabled + self.mock_airflow_url.__bool__.return_value = False + response = self.client.post(url, content_type='application/json', data=json.dumps(body)) + self.assertEqual(response.status_code, 403) diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index 7a2566e415..1100010398 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -625,8 +625,10 @@ def setUp(self): patcher = mock.patch('seqr.views.utils.airflow_utils.AuthorizedSession', mock.Mock(return_value=requests)) self.mock_authorized_session = patcher.start() self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.views.utils.airflow_utils.AIRFLOW_WEBSERVER_URL', self.MOCK_AIRFLOW_URL) - patcher.start() + patcher = mock.patch('seqr.views.utils.airflow_utils.AIRFLOW_WEBSERVER_URL') + self.mock_airflow_url = patcher.start() + self.mock_airflow_url.__str__.return_value = self.MOCK_AIRFLOW_URL + self.mock_airflow_url.__bool__.return_value = True self.addCleanup(patcher.stop) patcher = mock.patch('seqr.views.utils.airflow_utils.safe_post_to_slack') self.mock_slack = patcher.start() From 0aa0adf5b62e1e04b23fd600132aafb8a656ce91 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 Aug 2025 11:55:49 -0400 Subject: [PATCH 32/46] dag specific tests for clickhouse --- seqr/views/apis/data_manager_api_tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 6e983d1895..07f6d86c27 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -2015,6 +2015,13 @@ def _add_update_check_dag_responses(self, variables=None, **kwargs): return super()._add_update_check_dag_responses(**kwargs) + def _assert_expected_dag_trigger(self, response, dag_id, variables): + if dag_id == 'DELETE_PROJECTS': + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), {'info': ['Deleted all SNV_INDEL search data for project 1kg project n\xe5me with uni\xe7\xf8de']}) + else: + super()._assert_expected_dag_trigger(response, dag_id, variables) + def _assert_expected_airtable_errors(self, url): responses.replace( responses.GET, 'https://api.airtable.com/v0/app3Y97xtbbaOopVR/Samples', From 79eb2303124a7460f1594ad1265bb74c5764f55b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 Aug 2025 12:01:17 -0400 Subject: [PATCH 33/46] test helper name cleanup --- seqr/views/apis/data_manager_api_tests.py | 25 ++++++++++------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 07f6d86c27..a48b4e5507 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1519,7 +1519,7 @@ def test_trigger_delete_project(self): url = reverse(trigger_delete_project) self.check_data_manager_login(url) - self._test_trigger_single_dag( + self._test_trigger_search_data_update( url, 'DELETE_PROJECTS', {'project': PROJECT_GUID, 'datasetType': 'SNV_INDEL'}, @@ -1535,7 +1535,7 @@ def test_trigger_delete_family(self): url = reverse(trigger_delete_family) self.check_data_manager_login(url) - self._test_trigger_single_dag( + self._test_trigger_search_data_update( url, 'DELETE_FAMILIES', {'family': 'F000012_12', 'datasetType': 'MITO'}, @@ -1553,17 +1553,17 @@ def test_trigger_update_search_reference_data(self): self.check_data_manager_login(url) body = {'genomeVersion': '38', 'datasetType': 'SV'} - self._test_trigger_single_dag(url, 'UPDATE_REFERENCE_DATASETS', body, { + self._test_trigger_search_data_update(url, 'UPDATE_REFERENCE_DATASETS', body, { 'projects_to_run': None, 'dataset_type': 'SV', 'reference_genome': 'GRCh38', }) - def _test_trigger_single_dag(self, url, dag_id, body, dag_variables): + def _test_trigger_search_data_update(self, url, dag_id, body, dag_variables): response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self._assert_expected_dag_trigger(response, dag_id, dag_variables) + self._assert_expected_search_data_update(response, dag_id, dag_variables) - def _assert_expected_dag_trigger(self, response, dag_id, variables): + def _assert_expected_search_data_update(self, response, dag_id, variables): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'errors': ['This functionality is not available in the current search backend'], 'warnings': None}) @@ -1700,9 +1700,6 @@ def _assert_write_pedigree_error(self, response): def _test_validate_dataset_type(self, url): pass - def set_up_one_dag(self, *args, **kwargs): - pass - @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') class AnvilDataManagerAPITest(AirflowTestCase, DataManagerAPITest): @@ -2015,12 +2012,12 @@ def _add_update_check_dag_responses(self, variables=None, **kwargs): return super()._add_update_check_dag_responses(**kwargs) - def _assert_expected_dag_trigger(self, response, dag_id, variables): + def _assert_expected_search_data_update(self, response, dag_id, variables): if dag_id == 'DELETE_PROJECTS': self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), {'info': ['Deleted all SNV_INDEL search data for project 1kg project n\xe5me with uni\xe7\xf8de']}) else: - super()._assert_expected_dag_trigger(response, dag_id, variables) + super()._assert_expected_search_data_update(response, dag_id, variables) def _assert_expected_airtable_errors(self, url): responses.replace( @@ -2050,17 +2047,17 @@ def set_up_one_dag(self, dag_id=None, **kwargs): self.DAG_NAME = dag_id super().set_up_one_dag(**kwargs) - def _assert_expected_dag_trigger(self, response, dag_id, variables): + def _assert_expected_search_data_update(self, response, dag_id, variables): self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), {'info': [f'Triggered DAG {dag_id} with variables: {json.dumps(variables)}']}) self.assert_airflow_calls(variables, 5) - def _test_trigger_single_dag(self, url, dag_id, body, dag_variables): + def _test_trigger_search_data_update(self, url, dag_id, body, dag_variables): responses.calls.reset() self.set_up_one_dag(dag_id, variables=dag_variables) - super()._test_trigger_single_dag(url, dag_id, body, dag_variables) + super()._test_trigger_search_data_update(url, dag_id, body, dag_variables) self.set_dag_trigger_error_response() response = self.client.post(url, content_type='application/json', data=json.dumps(body)) From 871df628829c8570b8b6c21e9e41ff6bca3e69b5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 Aug 2025 12:57:35 -0400 Subject: [PATCH 34/46] actually test dropped proejct --- clickhouse_search/search.py | 9 +++--- seqr/views/apis/data_manager_api_tests.py | 35 ++++++++++++++++++++--- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 3216235123..851d3b1a66 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -537,9 +537,10 @@ def get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids, revers def delete_clickhouse_project(project, dataset_type=None, **kwargs): table_base = f'{GENOME_VERSION_LOOKUP[project.genome_version]}/{dataset_type}' with connections['clickhouse_write'].cursor() as cursor: - cursor.execute('ALTER TABLE %s DROP PARTITION %s', [f'{table_base}/entries', project.guid]) + cursor.execute(f'ALTER TABLE "{table_base}/entries" DROP PARTITION %s', [project.guid]) + cursor.execute(f'ALTER TABLE "{table_base}/project_gt_stats" DROP PARTITION %s', [project.guid]) view_name = f'{table_base}/project_gt_stats_to_gt_stats_mv' - cursor.execute('SYSTEM REFRESH VIEW %s', [view_name]) - cursor.execute('SYSTEM WAIT VIEW %s', [view_name]) - cursor.execute('SYSTEM RELOAD DICTIONARY %s', [f'{table_base}/gt_stats_dict']) + cursor.execute(f'SYSTEM REFRESH VIEW "{view_name}"') + cursor.execute(f'SYSTEM WAIT VIEW "{view_name}"') + cursor.execute(f'SYSTEM RELOAD DICTIONARY "{table_base}/gt_stats_dict"') return f'Deleted all {dataset_type} search data for project {project.name}' diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index a48b4e5507..0740dda503 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -6,6 +6,7 @@ from requests import HTTPError import responses +from clickhouse_search.models import EntriesSnvIndel, ProjectGtStatsSnvIndel, AnnotationsSnvIndel from seqr.utils.communication_utils import _set_bulk_notification_stream from seqr.views.apis.data_manager_api import elasticsearch_status, delete_index, \ update_rna_seq, load_rna_seq_sample_data, load_phenotype_prioritization_data, validate_callset, loading_vcfs, \ @@ -1560,9 +1561,13 @@ def test_trigger_update_search_reference_data(self): }) def _test_trigger_search_data_update(self, url, dag_id, body, dag_variables): + self._setup_trigger_search_data_update(url, dag_id, body, dag_variables) response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self._assert_expected_search_data_update(response, dag_id, dag_variables) + def _setup_trigger_search_data_update(self, *args, **kwargs): + return + def _assert_expected_search_data_update(self, response, dag_id, variables): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'errors': ['This functionality is not available in the current search backend'], 'warnings': None}) @@ -1703,7 +1708,7 @@ def _test_validate_dataset_type(self, url): @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') class AnvilDataManagerAPITest(AirflowTestCase, DataManagerAPITest): - fixtures = ['users', 'social_auth', '1kg_project', 'reference_data'] + fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'clickhouse_search'] LOADING_PROJECT_GUID = NON_ANALYST_PROJECT_GUID CALLSET_DIR = 'gs://test_bucket' @@ -2012,13 +2017,34 @@ def _add_update_check_dag_responses(self, variables=None, **kwargs): return super()._add_update_check_dag_responses(**kwargs) + def _setup_trigger_search_data_update(self, *args, **kwargs): + Project.objects.filter(guid=PROJECT_GUID).update(genome_version='38') + def _assert_expected_search_data_update(self, response, dag_id, variables): if dag_id == 'DELETE_PROJECTS': - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.json(), {'info': ['Deleted all SNV_INDEL search data for project 1kg project n\xe5me with uni\xe7\xf8de']}) + self._assert_expected_delete_project(response) else: super()._assert_expected_search_data_update(response, dag_id, variables) + def _assert_expected_delete_project(self, response): + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), { + 'info': ['Deleted all SNV_INDEL search data for project 1kg project n\xe5me with uni\xe7\xf8de'], + }) + self.assertEqual(EntriesSnvIndel.objects.filter(project_guid=PROJECT_GUID).count(), 0) + self.assertEqual(ProjectGtStatsSnvIndel.objects.filter(project_guid=PROJECT_GUID).count(), 0) + + updated_seqr_pops_by_key = dict(AnnotationsSnvIndel.objects.all().join_seqr_pop().values_list('key', 'seqrPop')) + self.assertDictEqual(updated_seqr_pops_by_key, { + 1: (2, 2, 1, 1), + 2: (1, 1, 0, 0), + 3: (0, 0, 0, 0), + 4: (0, 0, 0, 0), + 5: (1, 1, 0, 0), + 6: (0, 0, 0, 0), + 22: (0, 3, 0, 1), + }) + def _assert_expected_airtable_errors(self, url): responses.replace( responses.GET, 'https://api.airtable.com/v0/app3Y97xtbbaOopVR/Samples', @@ -2053,10 +2079,11 @@ def _assert_expected_search_data_update(self, response, dag_id, variables): self.assert_airflow_calls(variables, 5) - def _test_trigger_search_data_update(self, url, dag_id, body, dag_variables): + def _setup_trigger_search_data_update(self, url, dag_id, body, dag_variables): responses.calls.reset() self.set_up_one_dag(dag_id, variables=dag_variables) + def _test_trigger_search_data_update(self, url, dag_id, body, dag_variables): super()._test_trigger_search_data_update(url, dag_id, body, dag_variables) self.set_dag_trigger_error_response() From 0b8c3a487c217bd1f9a46355598e2c4086c3c334 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 Aug 2025 13:00:13 -0400 Subject: [PATCH 35/46] undo debug --- ui/pages/DataManagement/DataManagement.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/pages/DataManagement/DataManagement.jsx b/ui/pages/DataManagement/DataManagement.jsx index db2e0db7de..600e9044ac 100644 --- a/ui/pages/DataManagement/DataManagement.jsx +++ b/ui/pages/DataManagement/DataManagement.jsx @@ -61,7 +61,7 @@ const dataManagementPages = (user, elasticsearchEnabled) => { if (elasticsearchEnabled) { return ES_DATA_MANAGEMENT_PAGES } - return !user.isAnvil ? AIRFLOW_HAIL_SEARCH_DATA_MANAGEMENT_PAGES : LOCAL_HAIL_SEARCH_DATA_MANAGEMENT_PAGES + return user.isAnvil ? AIRFLOW_HAIL_SEARCH_DATA_MANAGEMENT_PAGES : LOCAL_HAIL_SEARCH_DATA_MANAGEMENT_PAGES } const DataManagement = ({ match, user, pages }) => ( From 8c9168c3ef8b38d479eb991297829d4f9d1c063e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 Aug 2025 13:50:16 -0400 Subject: [PATCH 36/46] codacy fix --- seqr/views/apis/data_manager_api_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 0740dda503..8e14871f70 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1561,7 +1561,7 @@ def test_trigger_update_search_reference_data(self): }) def _test_trigger_search_data_update(self, url, dag_id, body, dag_variables): - self._setup_trigger_search_data_update(url, dag_id, body, dag_variables) + self._setup_trigger_search_data_update(dag_id=dag_id, dag_variables=dag_variables) response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self._assert_expected_search_data_update(response, dag_id, dag_variables) @@ -2079,7 +2079,7 @@ def _assert_expected_search_data_update(self, response, dag_id, variables): self.assert_airflow_calls(variables, 5) - def _setup_trigger_search_data_update(self, url, dag_id, body, dag_variables): + def _setup_trigger_search_data_update(self, *args, dag_id=None, dag_variables=None, **kwargs): responses.calls.reset() self.set_up_one_dag(dag_id, variables=dag_variables) From 0c2c1fb548a3f5fa7986e40531442155d4566a17 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 Aug 2025 16:13:15 -0400 Subject: [PATCH 37/46] update clickhouse env vars --- .github/workflows/vlm-unit-tests.yaml | 4 ++-- vlm/clickhouse_utils.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 56774ef679..35c1c631e8 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -75,8 +75,8 @@ jobs: export NODE_ID=TestVLM export CLICKHOUSE_SERVICE_HOSTNAME=clickhouse export CLICKHOUSE_SERVICE_PORT=8123 - export CLICKHOUSE_USERNAME=vlm_test_user - export CLICKHOUSE_PASSWORD=vlm_test_password + export CLICKHOUSE_VLM_USERNAME=vlm_test_user + export CLICKHOUSE_VLM_PASSWORD=vlm_test_password export CLICKHOUSE_DATABASE=test_seqr coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py","./vlm/hail_backend_utils.py" -m pytest vlm/ coverage report --fail-under=95 diff --git a/vlm/clickhouse_utils.py b/vlm/clickhouse_utils.py index 6d5d0553be..dd32573776 100644 --- a/vlm/clickhouse_utils.py +++ b/vlm/clickhouse_utils.py @@ -5,8 +5,8 @@ CLICKHOUSE_CONNECTION_PARAMS = { 'host': os.environ.get('CLICKHOUSE_SERVICE_HOSTNAME'), 'port': os.environ.get('CLICKHOUSE_SERVICE_PORT'), - 'username': os.environ.get('CLICKHOUSE_USERNAME'), - 'password': os.environ.get('CLICKHOUSE_PASSWORD'), + 'username': os.environ.get('CLICKHOUSE_VLM_USERNAME'), + 'password': os.environ.get('CLICKHOUSE_VLM_PASSWORD'), 'database': os.environ.get('CLICKHOUSE_DATABASE', 'seqr'), } From ef387de831e798bc1b438f2e05e97fa5fa850572 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 Aug 2025 13:20:51 -0400 Subject: [PATCH 38/46] Revert "Merge pull request #4916 from broadinstitute/benb/join_table_non_snv_indel" This reverts commit 958c874b0e4a465b1dc6ac0b749681c0e8b2710a, reversing changes made to 2729f2fb0713915169db82acbe6f56bd4593be36. --- ...012_delete_annotationsdiskgcnv_and_more.py | 120 --------------- clickhouse_search/models.py | 138 ++++++++++++++---- 2 files changed, 106 insertions(+), 152 deletions(-) delete mode 100644 clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py diff --git a/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py b/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py deleted file mode 100644 index d1fea85abc..0000000000 --- a/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py +++ /dev/null @@ -1,120 +0,0 @@ -# Generated by Django 4.2.21 on 2025-08-06 00:31 - -import clickhouse_backend.models -import clickhouse_search.backend.fields -import clickhouse_search.backend.engines -from django.db import migrations -import django.db.models.manager - - -class Migration(migrations.Migration): - - dependencies = [ - ('clickhouse_search', '0011_clinvar_materialized_view'), - ] - - operations = [ - migrations.DeleteModel( - name='AnnotationsDiskGcnv', - ), - migrations.DeleteModel( - name='AnnotationsDiskMito', - ), - migrations.DeleteModel( - name='AnnotationsDiskSv', - ), - migrations.DeleteModel( - name='AnnotationsGcnv', - ), - migrations.DeleteModel( - name='AnnotationsMito', - ), - migrations.DeleteModel( - name='AnnotationsSv', - ), - migrations.CreateModel( - name='AnnotationsMito', - fields=[ - ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), - ('xpos', clickhouse_backend.models.UInt64Field()), - ('pos', clickhouse_backend.models.UInt32Field()), - ('ref', clickhouse_backend.models.StringField()), - ('alt', clickhouse_backend.models.StringField()), - ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), - ('rsid', clickhouse_backend.models.StringField(blank=True, null=True)), - ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), - ('common_low_heteroplasmy', clickhouse_backend.models.BoolField(blank=True, db_column='commonLowHeteroplasmy', null=True)), - ('mitomap_pathogenic', clickhouse_backend.models.BoolField(blank=True, db_column='mitomapPathogenic', null=True)), - ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('apogee', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('haplogroup_defining', clickhouse_backend.models.BoolField(null=True, blank=True)), ('hmtvar', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mitotip', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'likely_pathogenic'), (1, 'possibly_pathogenic'), (2, 'possibly_benign'), (3, 'likely_benign')], null=True)), ('mut_taster', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'D'), (1, 'A'), (2, 'N'), (3, 'P')], null=True)), ('sift', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mlc', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), - ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_mito', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('gnomad_mito_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))])), ('helix', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('helix_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))]))])), - ('sorted_transcript_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('aminoAcids', clickhouse_backend.models.StringField(blank=True, null=True)), ('biotype', clickhouse_backend.models.StringField(blank=True, null=True)), ('canonical', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('codons', clickhouse_backend.models.StringField(blank=True, null=True)), ('consequenceTerms', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True))), ('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsc', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsp', clickhouse_backend.models.StringField(blank=True, null=True)), ('loftee', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('isLofNagnag', clickhouse_backend.models.BoolField(blank=True, null=True)), ('lofFilters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(blank=True, null=True)))])), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True)), ('transcriptId', clickhouse_backend.models.StringField()), ('transcriptRank', clickhouse_backend.models.UInt8Field())], db_column='sortedTranscriptConsequences', group_by_key='geneId')), - ], - options={ - 'db_table': 'GRCh38/MITO/annotations_memory', - 'engine': clickhouse_search.backend.engines.Join('ALL', 'INNER', 'key', join_use_nulls=1, flatten_nested=0), - }, - managers=[ - ('objects', django.db.models.manager.Manager()), - ('_overwrite_base_manager', django.db.models.manager.Manager()), - ], - ), - migrations.CreateModel( - name='AnnotationsSv', - fields=[ - ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), - ('xpos', clickhouse_backend.models.UInt64Field()), - ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), - ('pos', clickhouse_backend.models.UInt32Field()), - ('end', clickhouse_backend.models.UInt32Field()), - ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('position', clickhouse_backend.models.UInt32Field(null=True, blank=True))], db_column='rg37LocusEnd')), - ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), - ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), - ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), - ('algorithms', clickhouse_backend.models.StringField(low_cardinality=True)), - ('bothsides_support', clickhouse_backend.models.BoolField(db_column='bothsidesSupport')), - ('cpx_intervals', clickhouse_search.backend.fields.NestedField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('start', clickhouse_backend.models.UInt32Field()), ('end', clickhouse_backend.models.UInt32Field()), ('type', clickhouse_backend.models.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')]))], db_column='cpxIntervals')), - ('end_chrom', clickhouse_backend.models.Enum8Field(null=True, blank=True, db_column='endChrom', choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), - ('sv_source_detail', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')]))], db_column='svSourceDetail')), - ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), - ('sv_type_detail', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, 'INS_iDEL'), (2, 'INVdel'), (3, 'INVdup'), (4, 'ME'), (5, 'ME:ALU'), (6, 'ME:LINE1'), (7, 'ME:SVA'), (8, 'dDUP'), (9, 'dDUP_iDEL'), (10, 'delINV'), (11, 'delINVdel'), (12, 'delINVdup'), (13, 'dupINV'), (14, 'dupINVdel'), (15, 'dupINVdup')], db_column='svTypeDetail')), - ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), - ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_svs', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field()), ('id', clickhouse_backend.models.StringField())]))])), - ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), - ], - options={ - 'db_table': 'GRCh38/SV/annotations_memory', - 'engine': clickhouse_search.backend.engines.Join('ALL', 'INNER', 'key', join_use_nulls=1, flatten_nested=0), - }, - managers=[ - ('objects', django.db.models.manager.Manager()), - ('_overwrite_base_manager', django.db.models.manager.Manager()), - ], - ), - migrations.CreateModel( - name='AnnotationsGcnv', - fields=[ - ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), - ('xpos', clickhouse_backend.models.UInt64Field()), - ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), - ('pos', clickhouse_backend.models.UInt32Field()), - ('end', clickhouse_backend.models.UInt32Field()), - ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('position', clickhouse_backend.models.UInt32Field(null=True, blank=True))], db_column='rg37LocusEnd')), - ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), - ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), - ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), - ('num_exon', clickhouse_backend.models.UInt16Field(db_column='numExon')), - ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), - ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), - ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sv_callset', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field())]))])), - ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), - ], - options={ - 'db_table': 'GRCh38/GCNV/annotations_memory', - 'engine': clickhouse_search.backend.engines.Join('ALL', 'INNER', 'key', join_use_nulls=1, flatten_nested=0), - }, - managers=[ - ('objects', django.db.models.manager.Manager()), - ('_overwrite_base_manager', django.db.models.manager.Manager()), - ], - ), - ] diff --git a/clickhouse_search/models.py b/clickhouse_search/models.py index 9432df87f7..cd9ea1cc44 100644 --- a/clickhouse_search/models.py +++ b/clickhouse_search/models.py @@ -57,7 +57,7 @@ def db_for_write(self, model, **hints): return None def allow_migrate(self, db, app_label, model_name=None, **hints): - if app_label == 'clickhouse_search' or hints.get('clickhouse'): + if f'{app_label}.{model_name}' in self.route_model_names or hints.get('clickhouse'): return db == 'clickhouse_write' elif db in {'clickhouse', 'clickhouse_write'}: return False @@ -72,28 +72,7 @@ def __init__(self, name, select='*', order_by=None): self.order_by = order_by -class FixtureLoadableClickhouseModel(models.ClickhouseModel): - - def _save_table( - self, - raw=False, - cls=None, - force_insert=False, - force_update=False, - using=None, - update_fields=None, - ): - # loaddata attempts to run an ALTER TABLE to update existing rows, but since primary keys can not be altered - # and JOIN tables can not be altered this command fails so need to use the force_insert flag to run an INSERT instead - return super()._save_table( - raw=raw, cls=cls, force_insert=True, force_update=force_update, using=using, update_fields=update_fields, - ) - - class Meta: - abstract = True - - -class BaseAnnotations(FixtureLoadableClickhouseModel): +class BaseAnnotations(models.ClickhouseModel): CHROMOSOME_CHOICES = [(i+1, chrom) for i, chrom in enumerate(CHROMOSOMES)] SEQR_POPULATIONS = [ @@ -302,7 +281,7 @@ class Meta: db_table = 'GRCh38/SNV_INDEL/annotations_disk' engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/SNV_INDEL/annotations', primary_key='key', flatten_nested=0) -class AnnotationsMito(BaseAnnotationsMitoSnvIndel): +class BaseAnnotationsMito(BaseAnnotationsMitoSnvIndel): ANNOTATION_CONSTANTS = { 'chrom': 'M', 'liftedOverChrom': 'MT', @@ -358,11 +337,23 @@ class AnnotationsMito(BaseAnnotationsMitoSnvIndel): populations = NamedTupleField(POPULATION_FIELDS) sorted_transcript_consequences = NestedField(BaseAnnotationsMitoSnvIndel.TRANSCRIPTS_FIELDS, db_column='sortedTranscriptConsequences', group_by_key='geneId') + class Meta: + abstract = True + +class AnnotationsMito(BaseAnnotationsMito): + class Meta: db_table = 'GRCh38/MITO/annotations_memory' - engine = Join('ALL', 'INNER', 'key', join_use_nulls=1, flatten_nested=0) + engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/MITO/annotations', primary_key='key', flatten_nested=0) + +class AnnotationsDiskMito(BaseAnnotationsMito): -class AnnotationsSv(BaseAnnotationsSvGcnv): + class Meta: + db_table = 'GRCh38/MITO/annotations_disk' + engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/MITO/annotations', primary_key='key', flatten_nested=0) + + +class BaseAnnotationsSv(BaseAnnotationsSvGcnv): POPULATION_FIELDS = [ ('gnomad_svs', NamedTupleField([ ('af', models.DecimalField(max_digits=9, decimal_places=5)), @@ -390,11 +381,22 @@ class AnnotationsSv(BaseAnnotationsSvGcnv): sv_type_detail = models.Enum8Field(db_column='svTypeDetail', return_int=False, choices=SV_TYPE_DETAILS, null=True, blank=True) populations = NamedTupleField(POPULATION_FIELDS) + class Meta: + abstract = True + +class AnnotationsSv(BaseAnnotationsSv): + class Meta: db_table = 'GRCh38/SV/annotations_memory' - engine = Join('ALL', 'INNER', 'key', join_use_nulls=1, flatten_nested=0) + engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/SV/annotations', primary_key='key', flatten_nested=0) -class AnnotationsGcnv(BaseAnnotationsSvGcnv): +class AnnotationsDiskSv(BaseAnnotationsSv): + + class Meta: + db_table = 'GRCh38/SV/annotations_disk' + engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/SV/annotations', primary_key='key', flatten_nested=0) + +class BaseAnnotationsGcnv(BaseAnnotationsSvGcnv): POPULATION_FIELDS = [ ('sv_callset', NamedTupleField([ ('ac', models.UInt32Field()), @@ -416,11 +418,23 @@ class AnnotationsGcnv(BaseAnnotationsSvGcnv): num_exon = models.UInt16Field(db_column='numExon') populations = NamedTupleField(POPULATION_FIELDS) + class Meta: + abstract = True + +class AnnotationsGcnv(BaseAnnotationsGcnv): + class Meta: db_table = 'GRCh38/GCNV/annotations_memory' - engine = Join('ALL', 'LEFT', 'key', join_use_nulls=1, flatten_nested=0) + engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/GCNV/annotations', primary_key='key', flatten_nested=0) + +class AnnotationsDiskGcnv(BaseAnnotationsGcnv): + + class Meta: + db_table = 'GRCh38/GCNV/annotations_disk' + engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/GCNV/annotations', primary_key='key', flatten_nested=0) + -class BaseClinvar(FixtureLoadableClickhouseModel): +class BaseClinvar(models.ClickhouseModel): CLINVAR_ASSERTIONS = [ 'Affects', @@ -479,6 +493,21 @@ class BaseClinvarAllVariants(BaseClinvar): version = models.DateField() variant_id = models.StringField(db_column='variantId', primary_key=True) + def _save_table( + self, + raw=False, + cls=None, + force_insert=False, + force_update=False, + using=None, + update_fields=None, + ): + # loaddata attempts to run an ALTER TABLE to update existing rows, but since JOIN tables can not be altered + # this command fails so need to use the force_insert flag to run an INSERT instead + return super()._save_table( + raw=raw, cls=cls, force_insert=True, force_update=force_update, using=using, update_fields=update_fields, + ) + class Meta: abstract = True engine = models.MergeTree( @@ -501,6 +530,21 @@ class Meta(BaseClinvarAllVariants.Meta): class BaseClinvarJoin(BaseClinvar): + def _save_table( + self, + raw=False, + cls=None, + force_insert=False, + force_update=False, + using=None, + update_fields=None, + ): + # loaddata attempts to run an ALTER TABLE to update existing rows, but since JOIN tables can not be altered + # this command fails so need to use the force_insert flag to run an INSERT instead + return super()._save_table( + raw=raw, cls=cls, force_insert=True, force_update=force_update, using=using, update_fields=update_fields, + ) + class Meta: abstract = True engine = Join('ALL', 'LEFT', 'key', join_use_nulls=1, flatten_nested=0) @@ -522,7 +566,7 @@ class Meta(BaseClinvarJoin.Meta): db_table = 'GRCh38/MITO/clinvar' -class BaseEntries(FixtureLoadableClickhouseModel): +class BaseEntries(models.ClickhouseModel): project_guid = models.StringField(low_cardinality=True) family_guid = models.StringField() xpos = UInt64FieldDeltaCodecField() @@ -531,6 +575,21 @@ class BaseEntries(FixtureLoadableClickhouseModel): objects = EntriesManager.as_manager() + def _save_table( + self, + raw=False, + cls=None, + force_insert=False, + force_update=False, + using=None, + update_fields=None, + ): + # loaddata attempts to run an ALTER TABLE to update existing rows, but since primary keys can not be altered + # this command fails so need to use the force_insert flag to run an INSERT instead + return super()._save_table( + raw=raw, cls=cls, force_insert=True, force_update=force_update, using=using, update_fields=update_fields, + ) + class Meta: abstract = True engine = CollapsingMergeTree( @@ -710,12 +769,27 @@ class Meta: db_table = 'GRCh38/SNV_INDEL/transcripts' engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/SNV_INDEL/transcripts', primary_key='key', flatten_nested=0) -class BaseKeyLookup(FixtureLoadableClickhouseModel): +class BaseKeyLookup(models.ClickhouseModel): variant_id = models.StringField(db_column='variantId', primary_key=True) class Meta: abstract = True + def _save_table( + self, + raw=False, + cls=None, + force_insert=False, + force_update=False, + using=None, + update_fields=None, + ): + # loaddata attempts to run an ALTER TABLE to update existing rows, but since primary keys can not be altered + # this command fails so need to use the force_insert flag to run an INSERT instead + return super()._save_table( + raw=raw, cls=cls, force_insert=True, force_update=force_update, using=using, update_fields=update_fields, + ) + class KeyLookupGRCh37SnvIndel(BaseKeyLookup): key = OneToOneField('AnnotationsGRCh37SnvIndel', db_column='key', on_delete=CASCADE) From ee529af78dcc89b69cd9cbed1b033fd063494790 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 Aug 2025 13:26:57 -0400 Subject: [PATCH 39/46] migrations --- ...012_delete_annotationsdiskgcnv_and_more.py | 120 +++++++++++ ...nsdiskgcnv_annotationsdiskmito_and_more.py | 196 ++++++++++++++++++ clickhouse_search/models.py | 2 +- 3 files changed, 317 insertions(+), 1 deletion(-) create mode 100644 clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py create mode 100644 clickhouse_search/migrations/0013_annotationsdiskgcnv_annotationsdiskmito_and_more.py diff --git a/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py b/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py new file mode 100644 index 0000000000..d1fea85abc --- /dev/null +++ b/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py @@ -0,0 +1,120 @@ +# Generated by Django 4.2.21 on 2025-08-06 00:31 + +import clickhouse_backend.models +import clickhouse_search.backend.fields +import clickhouse_search.backend.engines +from django.db import migrations +import django.db.models.manager + + +class Migration(migrations.Migration): + + dependencies = [ + ('clickhouse_search', '0011_clinvar_materialized_view'), + ] + + operations = [ + migrations.DeleteModel( + name='AnnotationsDiskGcnv', + ), + migrations.DeleteModel( + name='AnnotationsDiskMito', + ), + migrations.DeleteModel( + name='AnnotationsDiskSv', + ), + migrations.DeleteModel( + name='AnnotationsGcnv', + ), + migrations.DeleteModel( + name='AnnotationsMito', + ), + migrations.DeleteModel( + name='AnnotationsSv', + ), + migrations.CreateModel( + name='AnnotationsMito', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('pos', clickhouse_backend.models.UInt32Field()), + ('ref', clickhouse_backend.models.StringField()), + ('alt', clickhouse_backend.models.StringField()), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('rsid', clickhouse_backend.models.StringField(blank=True, null=True)), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('common_low_heteroplasmy', clickhouse_backend.models.BoolField(blank=True, db_column='commonLowHeteroplasmy', null=True)), + ('mitomap_pathogenic', clickhouse_backend.models.BoolField(blank=True, db_column='mitomapPathogenic', null=True)), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('apogee', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('haplogroup_defining', clickhouse_backend.models.BoolField(null=True, blank=True)), ('hmtvar', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mitotip', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'likely_pathogenic'), (1, 'possibly_pathogenic'), (2, 'possibly_benign'), (3, 'likely_benign')], null=True)), ('mut_taster', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'D'), (1, 'A'), (2, 'N'), (3, 'P')], null=True)), ('sift', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mlc', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_mito', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('gnomad_mito_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))])), ('helix', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('helix_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))]))])), + ('sorted_transcript_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('aminoAcids', clickhouse_backend.models.StringField(blank=True, null=True)), ('biotype', clickhouse_backend.models.StringField(blank=True, null=True)), ('canonical', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('codons', clickhouse_backend.models.StringField(blank=True, null=True)), ('consequenceTerms', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True))), ('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsc', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsp', clickhouse_backend.models.StringField(blank=True, null=True)), ('loftee', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('isLofNagnag', clickhouse_backend.models.BoolField(blank=True, null=True)), ('lofFilters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(blank=True, null=True)))])), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True)), ('transcriptId', clickhouse_backend.models.StringField()), ('transcriptRank', clickhouse_backend.models.UInt8Field())], db_column='sortedTranscriptConsequences', group_by_key='geneId')), + ], + options={ + 'db_table': 'GRCh38/MITO/annotations_memory', + 'engine': clickhouse_search.backend.engines.Join('ALL', 'INNER', 'key', join_use_nulls=1, flatten_nested=0), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsSv', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('pos', clickhouse_backend.models.UInt32Field()), + ('end', clickhouse_backend.models.UInt32Field()), + ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('position', clickhouse_backend.models.UInt32Field(null=True, blank=True))], db_column='rg37LocusEnd')), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('algorithms', clickhouse_backend.models.StringField(low_cardinality=True)), + ('bothsides_support', clickhouse_backend.models.BoolField(db_column='bothsidesSupport')), + ('cpx_intervals', clickhouse_search.backend.fields.NestedField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('start', clickhouse_backend.models.UInt32Field()), ('end', clickhouse_backend.models.UInt32Field()), ('type', clickhouse_backend.models.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')]))], db_column='cpxIntervals')), + ('end_chrom', clickhouse_backend.models.Enum8Field(null=True, blank=True, db_column='endChrom', choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('sv_source_detail', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')]))], db_column='svSourceDetail')), + ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), + ('sv_type_detail', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, 'INS_iDEL'), (2, 'INVdel'), (3, 'INVdup'), (4, 'ME'), (5, 'ME:ALU'), (6, 'ME:LINE1'), (7, 'ME:SVA'), (8, 'dDUP'), (9, 'dDUP_iDEL'), (10, 'delINV'), (11, 'delINVdel'), (12, 'delINVdup'), (13, 'dupINV'), (14, 'dupINVdel'), (15, 'dupINVdup')], db_column='svTypeDetail')), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_svs', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field()), ('id', clickhouse_backend.models.StringField())]))])), + ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), + ], + options={ + 'db_table': 'GRCh38/SV/annotations_memory', + 'engine': clickhouse_search.backend.engines.Join('ALL', 'INNER', 'key', join_use_nulls=1, flatten_nested=0), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsGcnv', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('pos', clickhouse_backend.models.UInt32Field()), + ('end', clickhouse_backend.models.UInt32Field()), + ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('position', clickhouse_backend.models.UInt32Field(null=True, blank=True))], db_column='rg37LocusEnd')), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('num_exon', clickhouse_backend.models.UInt16Field(db_column='numExon')), + ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sv_callset', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field())]))])), + ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), + ], + options={ + 'db_table': 'GRCh38/GCNV/annotations_memory', + 'engine': clickhouse_search.backend.engines.Join('ALL', 'INNER', 'key', join_use_nulls=1, flatten_nested=0), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + ] diff --git a/clickhouse_search/migrations/0013_annotationsdiskgcnv_annotationsdiskmito_and_more.py b/clickhouse_search/migrations/0013_annotationsdiskgcnv_annotationsdiskmito_and_more.py new file mode 100644 index 0000000000..2f0926d315 --- /dev/null +++ b/clickhouse_search/migrations/0013_annotationsdiskgcnv_annotationsdiskmito_and_more.py @@ -0,0 +1,196 @@ +import clickhouse_backend.models +import clickhouse_search.backend.engines +import clickhouse_search.backend.fields +from django.db import migrations +import django.db.models.manager + +from settings import CLICKHOUSE_IN_MEMORY_DIR, CLICKHOUSE_DATA_DIR + + +class Migration(migrations.Migration): + + dependencies = [ + ('clickhouse_search', '0012_delete_annotationsdiskgcnv_and_more'), + ] + + operations = [ + migrations.DeleteModel( + name='AnnotationsGcnv', + ), + migrations.DeleteModel( + name='AnnotationsMito', + ), + migrations.DeleteModel( + name='AnnotationsSv', + ), + migrations.CreateModel( + name='AnnotationsGcnv', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('pos', clickhouse_backend.models.UInt32Field()), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('end', clickhouse_backend.models.UInt32Field()), + ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], null=True)), ('position', clickhouse_backend.models.UInt32Field(blank=True, null=True))], db_column='rg37LocusEnd')), + ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), + ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), + ('num_exon', clickhouse_backend.models.UInt16Field(db_column='numExon')), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sv_callset', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field())]))])), + ], + options={ + 'db_table': 'GRCh38/GCNV/annotations_memory', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsMito', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('pos', clickhouse_backend.models.UInt32Field()), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('ref', clickhouse_backend.models.StringField()), + ('alt', clickhouse_backend.models.StringField()), + ('rsid', clickhouse_backend.models.StringField(blank=True, null=True)), + ('common_low_heteroplasmy', clickhouse_backend.models.BoolField(blank=True, db_column='commonLowHeteroplasmy', null=True)), + ('mitomap_pathogenic', clickhouse_backend.models.BoolField(blank=True, db_column='mitomapPathogenic', null=True)), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('apogee', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('haplogroup_defining', clickhouse_backend.models.BoolField(blank=True, null=True)), ('hmtvar', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mitotip', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'likely_pathogenic'), (1, 'possibly_pathogenic'), (2, 'possibly_benign'), (3, 'likely_benign')], null=True)), ('mut_taster', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'D'), (1, 'A'), (2, 'N'), (3, 'P')], null=True)), ('sift', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mlc', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_mito', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('gnomad_mito_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))])), ('helix', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('helix_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))]))])), + ('sorted_transcript_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('aminoAcids', clickhouse_backend.models.StringField(blank=True, null=True)), ('biotype', clickhouse_backend.models.StringField(blank=True, null=True)), ('canonical', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('codons', clickhouse_backend.models.StringField(blank=True, null=True)), ('consequenceTerms', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True))), ('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsc', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsp', clickhouse_backend.models.StringField(blank=True, null=True)), ('loftee', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('isLofNagnag', clickhouse_backend.models.BoolField(blank=True, null=True)), ('lofFilters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(blank=True, null=True)))])), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True)), ('transcriptId', clickhouse_backend.models.StringField()), ('transcriptRank', clickhouse_backend.models.UInt8Field())], db_column='sortedTranscriptConsequences')), + ], + options={ + 'db_table': 'GRCh38/MITO/annotations_memory', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsSv', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('pos', clickhouse_backend.models.UInt32Field()), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('end', clickhouse_backend.models.UInt32Field()), + ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], null=True)), ('position', clickhouse_backend.models.UInt32Field(blank=True, null=True))], db_column='rg37LocusEnd')), + ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), + ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), + ('algorithms', clickhouse_backend.models.StringField(low_cardinality=True)), + ('bothsides_support', clickhouse_backend.models.BoolField(db_column='bothsidesSupport')), + ('cpx_intervals', clickhouse_search.backend.fields.NestedField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('start', clickhouse_backend.models.UInt32Field()), ('end', clickhouse_backend.models.UInt32Field()), ('type', clickhouse_backend.models.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')]))], db_column='cpxIntervals')), + ('end_chrom', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='endChrom', null=True)), + ('sv_source_detail', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], null=True))], db_column='svSourceDetail')), + ('sv_type_detail', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'INS_iDEL'), (2, 'INVdel'), (3, 'INVdup'), (4, 'ME'), (5, 'ME:ALU'), (6, 'ME:LINE1'), (7, 'ME:SVA'), (8, 'dDUP'), (9, 'dDUP_iDEL'), (10, 'delINV'), (11, 'delINVdel'), (12, 'delINVdup'), (13, 'dupINV'), (14, 'dupINVdel'), (15, 'dupINVdup')], db_column='svTypeDetail', null=True)), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_svs', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field()), ('id', clickhouse_backend.models.StringField())]))])), + ], + options={ + 'db_table': 'GRCh38/SV/annotations_memory', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsDiskGcnv', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('pos', clickhouse_backend.models.UInt32Field()), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('end', clickhouse_backend.models.UInt32Field()), + ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], null=True)), ('position', clickhouse_backend.models.UInt32Field(blank=True, null=True))], db_column='rg37LocusEnd')), + ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), + ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), + ('num_exon', clickhouse_backend.models.UInt16Field(db_column='numExon')), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sv_callset', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field())]))])), + ], + options={ + 'db_table': 'GRCh38/GCNV/annotations_disk', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsDiskMito', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('pos', clickhouse_backend.models.UInt32Field()), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('ref', clickhouse_backend.models.StringField()), + ('alt', clickhouse_backend.models.StringField()), + ('rsid', clickhouse_backend.models.StringField(blank=True, null=True)), + ('common_low_heteroplasmy', clickhouse_backend.models.BoolField(blank=True, db_column='commonLowHeteroplasmy', null=True)), + ('mitomap_pathogenic', clickhouse_backend.models.BoolField(blank=True, db_column='mitomapPathogenic', null=True)), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('apogee', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('haplogroup_defining', clickhouse_backend.models.BoolField(blank=True, null=True)), ('hmtvar', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mitotip', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'likely_pathogenic'), (1, 'possibly_pathogenic'), (2, 'possibly_benign'), (3, 'likely_benign')], null=True)), ('mut_taster', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'D'), (1, 'A'), (2, 'N'), (3, 'P')], null=True)), ('sift', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mlc', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_mito', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('gnomad_mito_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))])), ('helix', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('helix_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))]))])), + ('sorted_transcript_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('aminoAcids', clickhouse_backend.models.StringField(blank=True, null=True)), ('biotype', clickhouse_backend.models.StringField(blank=True, null=True)), ('canonical', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('codons', clickhouse_backend.models.StringField(blank=True, null=True)), ('consequenceTerms', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True))), ('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsc', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsp', clickhouse_backend.models.StringField(blank=True, null=True)), ('loftee', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('isLofNagnag', clickhouse_backend.models.BoolField(blank=True, null=True)), ('lofFilters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(blank=True, null=True)))])), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True)), ('transcriptId', clickhouse_backend.models.StringField()), ('transcriptRank', clickhouse_backend.models.UInt8Field())], db_column='sortedTranscriptConsequences')), + ], + options={ + 'db_table': 'GRCh38/MITO/annotations_disk', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsDiskSv', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('pos', clickhouse_backend.models.UInt32Field()), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('end', clickhouse_backend.models.UInt32Field()), + ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], null=True)), ('position', clickhouse_backend.models.UInt32Field(blank=True, null=True))], db_column='rg37LocusEnd')), + ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), + ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), + ('algorithms', clickhouse_backend.models.StringField(low_cardinality=True)), + ('bothsides_support', clickhouse_backend.models.BoolField(db_column='bothsidesSupport')), + ('cpx_intervals', clickhouse_search.backend.fields.NestedField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('start', clickhouse_backend.models.UInt32Field()), ('end', clickhouse_backend.models.UInt32Field()), ('type', clickhouse_backend.models.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')]))], db_column='cpxIntervals')), + ('end_chrom', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='endChrom', null=True)), + ('sv_source_detail', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], null=True))], db_column='svSourceDetail')), + ('sv_type_detail', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'INS_iDEL'), (2, 'INVdel'), (3, 'INVdup'), (4, 'ME'), (5, 'ME:ALU'), (6, 'ME:LINE1'), (7, 'ME:SVA'), (8, 'dDUP'), (9, 'dDUP_iDEL'), (10, 'delINV'), (11, 'delINVdel'), (12, 'delINVdup'), (13, 'dupINV'), (14, 'dupINVdel'), (15, 'dupINVdup')], db_column='svTypeDetail', null=True)), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_svs', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field()), ('id', clickhouse_backend.models.StringField())]))])), + ], + options={ + 'db_table': 'GRCh38/SV/annotations_disk', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + ] diff --git a/clickhouse_search/models.py b/clickhouse_search/models.py index cd9ea1cc44..b8b0668eba 100644 --- a/clickhouse_search/models.py +++ b/clickhouse_search/models.py @@ -57,7 +57,7 @@ def db_for_write(self, model, **hints): return None def allow_migrate(self, db, app_label, model_name=None, **hints): - if f'{app_label}.{model_name}' in self.route_model_names or hints.get('clickhouse'): + if app_label == 'clickhouse_search' or hints.get('clickhouse'): return db == 'clickhouse_write' elif db in {'clickhouse', 'clickhouse_write'}: return False From 7eceb90a4ec3f0a12e1082894e884fd54cfc7eb2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 Aug 2025 14:47:23 -0400 Subject: [PATCH 40/46] fix fistrue loading for new join tables --- clickhouse_search/models.py | 89 +++++++++++-------------------------- 1 file changed, 25 insertions(+), 64 deletions(-) diff --git a/clickhouse_search/models.py b/clickhouse_search/models.py index b8b0668eba..39bba48862 100644 --- a/clickhouse_search/models.py +++ b/clickhouse_search/models.py @@ -72,7 +72,28 @@ def __init__(self, name, select='*', order_by=None): self.order_by = order_by -class BaseAnnotations(models.ClickhouseModel): +class FixtureLoadableClickhouseModel(models.ClickhouseModel): + + def _save_table( + self, + raw=False, + cls=None, + force_insert=False, + force_update=False, + using=None, + update_fields=None, + ): + # loaddata attempts to run an ALTER TABLE to update existing rows, but since primary keys can not be altered + # this command fails so need to use the force_insert flag to run an INSERT instead + return super()._save_table( + raw=raw, cls=cls, force_insert=True, force_update=force_update, using=using, update_fields=update_fields, + ) + + class Meta: + abstract = True + + +class BaseAnnotations(FixtureLoadableClickhouseModel): CHROMOSOME_CHOICES = [(i+1, chrom) for i, chrom in enumerate(CHROMOSOMES)] SEQR_POPULATIONS = [ @@ -434,7 +455,7 @@ class Meta: engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/GCNV/annotations', primary_key='key', flatten_nested=0) -class BaseClinvar(models.ClickhouseModel): +class BaseClinvar(FixtureLoadableClickhouseModel): CLINVAR_ASSERTIONS = [ 'Affects', @@ -493,21 +514,6 @@ class BaseClinvarAllVariants(BaseClinvar): version = models.DateField() variant_id = models.StringField(db_column='variantId', primary_key=True) - def _save_table( - self, - raw=False, - cls=None, - force_insert=False, - force_update=False, - using=None, - update_fields=None, - ): - # loaddata attempts to run an ALTER TABLE to update existing rows, but since JOIN tables can not be altered - # this command fails so need to use the force_insert flag to run an INSERT instead - return super()._save_table( - raw=raw, cls=cls, force_insert=True, force_update=force_update, using=using, update_fields=update_fields, - ) - class Meta: abstract = True engine = models.MergeTree( @@ -530,21 +536,6 @@ class Meta(BaseClinvarAllVariants.Meta): class BaseClinvarJoin(BaseClinvar): - def _save_table( - self, - raw=False, - cls=None, - force_insert=False, - force_update=False, - using=None, - update_fields=None, - ): - # loaddata attempts to run an ALTER TABLE to update existing rows, but since JOIN tables can not be altered - # this command fails so need to use the force_insert flag to run an INSERT instead - return super()._save_table( - raw=raw, cls=cls, force_insert=True, force_update=force_update, using=using, update_fields=update_fields, - ) - class Meta: abstract = True engine = Join('ALL', 'LEFT', 'key', join_use_nulls=1, flatten_nested=0) @@ -566,7 +557,7 @@ class Meta(BaseClinvarJoin.Meta): db_table = 'GRCh38/MITO/clinvar' -class BaseEntries(models.ClickhouseModel): +class BaseEntries(FixtureLoadableClickhouseModel): project_guid = models.StringField(low_cardinality=True) family_guid = models.StringField() xpos = UInt64FieldDeltaCodecField() @@ -575,21 +566,6 @@ class BaseEntries(models.ClickhouseModel): objects = EntriesManager.as_manager() - def _save_table( - self, - raw=False, - cls=None, - force_insert=False, - force_update=False, - using=None, - update_fields=None, - ): - # loaddata attempts to run an ALTER TABLE to update existing rows, but since primary keys can not be altered - # this command fails so need to use the force_insert flag to run an INSERT instead - return super()._save_table( - raw=raw, cls=cls, force_insert=True, force_update=force_update, using=using, update_fields=update_fields, - ) - class Meta: abstract = True engine = CollapsingMergeTree( @@ -769,27 +745,12 @@ class Meta: db_table = 'GRCh38/SNV_INDEL/transcripts' engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/SNV_INDEL/transcripts', primary_key='key', flatten_nested=0) -class BaseKeyLookup(models.ClickhouseModel): +class BaseKeyLookup(FixtureLoadableClickhouseModel): variant_id = models.StringField(db_column='variantId', primary_key=True) class Meta: abstract = True - def _save_table( - self, - raw=False, - cls=None, - force_insert=False, - force_update=False, - using=None, - update_fields=None, - ): - # loaddata attempts to run an ALTER TABLE to update existing rows, but since primary keys can not be altered - # this command fails so need to use the force_insert flag to run an INSERT instead - return super()._save_table( - raw=raw, cls=cls, force_insert=True, force_update=force_update, using=using, update_fields=update_fields, - ) - class KeyLookupGRCh37SnvIndel(BaseKeyLookup): key = OneToOneField('AnnotationsGRCh37SnvIndel', db_column='key', on_delete=CASCADE) From 49f6874ca58816be8054535a54b5083add31ff25 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 Aug 2025 13:29:33 -0400 Subject: [PATCH 41/46] clean up --- clickhouse_search/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/models.py b/clickhouse_search/models.py index 39bba48862..6329351215 100644 --- a/clickhouse_search/models.py +++ b/clickhouse_search/models.py @@ -57,7 +57,7 @@ def db_for_write(self, model, **hints): return None def allow_migrate(self, db, app_label, model_name=None, **hints): - if app_label == 'clickhouse_search' or hints.get('clickhouse'): + if app_label == 'clickhouse_search' or hints.get('clickhouse'): return db == 'clickhouse_write' elif db in {'clickhouse', 'clickhouse_write'}: return False From 54fd2436e3ccf7a7059dc1681f73a236dff622f2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 Aug 2025 13:30:30 -0400 Subject: [PATCH 42/46] clean up --- clickhouse_search/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/models.py b/clickhouse_search/models.py index 6329351215..c23171abcf 100644 --- a/clickhouse_search/models.py +++ b/clickhouse_search/models.py @@ -84,7 +84,7 @@ def _save_table( update_fields=None, ): # loaddata attempts to run an ALTER TABLE to update existing rows, but since primary keys can not be altered - # this command fails so need to use the force_insert flag to run an INSERT instead + # and JOIN tables can not be altered this command fails so need to use the force_insert flag to run an INSERT instead return super()._save_table( raw=raw, cls=cls, force_insert=True, force_update=force_update, using=using, update_fields=update_fields, ) From f847b27bd60c3950956d6d043ff95c839b2f3890 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 Aug 2025 13:43:40 -0400 Subject: [PATCH 43/46] clean up migrations to prevent recreation of tables in empty db --- ...to_annotationsmito_entriesmito_and_more.py | 54 --------------- ...notationsdisksv_annotationssv_entriessv.py | 66 ------------------- ...nv_annotationsgcnv_entriesgcnv_and_more.py | 55 ---------------- ...012_delete_annotationsdiskgcnv_and_more.py | 18 ----- 4 files changed, 193 deletions(-) diff --git a/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py b/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py index a88e7cbf64..7cc402bffd 100644 --- a/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py +++ b/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py @@ -7,8 +7,6 @@ import django.db.models.deletion import django.db.models.manager -from settings import CLICKHOUSE_IN_MEMORY_DIR, CLICKHOUSE_DATA_DIR - class Migration(migrations.Migration): @@ -17,58 +15,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.CreateModel( - name='AnnotationsDiskMito', - fields=[ - ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), - ('xpos', clickhouse_backend.models.UInt64Field()), - ('pos', clickhouse_backend.models.UInt32Field()), - ('ref', clickhouse_backend.models.StringField()), - ('alt', clickhouse_backend.models.StringField()), - ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), - ('rsid', clickhouse_backend.models.StringField(blank=True, null=True)), - ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), - ('common_low_heteroplasmy', clickhouse_backend.models.BoolField(blank=True, db_column='commonLowHeteroplasmy', null=True)), - ('mitomap_pathogenic', clickhouse_backend.models.BoolField(blank=True, db_column='mitomapPathogenic', null=True)), - ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('apogee', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('haplogroup_defining', clickhouse_backend.models.BoolField(null=True, blank=True)), ('hmtvar', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mitotip', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'likely_pathogenic'), (1, 'possibly_pathogenic'), (2, 'possibly_benign'), (3, 'likely_benign')], null=True)), ('mut_taster', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'D'), (1, 'A'), (2, 'N'), (3, 'P')], null=True)), ('sift', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mlc', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), - ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_mito', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('gnomad_mito_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))])), ('helix', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('helix_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))]))])), - ('sorted_transcript_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('aminoAcids', clickhouse_backend.models.StringField(blank=True, null=True)), ('biotype', clickhouse_backend.models.StringField(blank=True, null=True)), ('canonical', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('codons', clickhouse_backend.models.StringField(blank=True, null=True)), ('consequenceTerms', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True))), ('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsc', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsp', clickhouse_backend.models.StringField(blank=True, null=True)), ('loftee', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('isLofNagnag', clickhouse_backend.models.BoolField(blank=True, null=True)), ('lofFilters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(blank=True, null=True)))])), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True)), ('transcriptId', clickhouse_backend.models.StringField()), ('transcriptRank', clickhouse_backend.models.UInt8Field())], db_column='sortedTranscriptConsequences', group_by_key='geneId')), - ], - options={ - 'db_table': 'GRCh38/MITO/annotations_disk', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), - }, - managers=[ - ('objects', django.db.models.manager.Manager()), - ('_overwrite_base_manager', django.db.models.manager.Manager()), - ], - ), - migrations.CreateModel( - name='AnnotationsMito', - fields=[ - ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), - ('xpos', clickhouse_backend.models.UInt64Field()), - ('pos', clickhouse_backend.models.UInt32Field()), - ('ref', clickhouse_backend.models.StringField()), - ('alt', clickhouse_backend.models.StringField()), - ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), - ('rsid', clickhouse_backend.models.StringField(blank=True, null=True)), - ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), - ('common_low_heteroplasmy', clickhouse_backend.models.BoolField(blank=True, db_column='commonLowHeteroplasmy', null=True)), - ('mitomap_pathogenic', clickhouse_backend.models.BoolField(blank=True, db_column='mitomapPathogenic', null=True)), - ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('apogee', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('haplogroup_defining', clickhouse_backend.models.BoolField(null=True, blank=True)), ('hmtvar', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mitotip', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'likely_pathogenic'), (1, 'possibly_pathogenic'), (2, 'possibly_benign'), (3, 'likely_benign')], null=True)), ('mut_taster', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'D'), (1, 'A'), (2, 'N'), (3, 'P')], null=True)), ('sift', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mlc', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), - ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_mito', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('gnomad_mito_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))])), ('helix', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('helix_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))]))])), - ('sorted_transcript_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('aminoAcids', clickhouse_backend.models.StringField(blank=True, null=True)), ('biotype', clickhouse_backend.models.StringField(blank=True, null=True)), ('canonical', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('codons', clickhouse_backend.models.StringField(blank=True, null=True)), ('consequenceTerms', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True))), ('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsc', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsp', clickhouse_backend.models.StringField(blank=True, null=True)), ('loftee', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('isLofNagnag', clickhouse_backend.models.BoolField(blank=True, null=True)), ('lofFilters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(blank=True, null=True)))])), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True)), ('transcriptId', clickhouse_backend.models.StringField()), ('transcriptRank', clickhouse_backend.models.UInt8Field())], db_column='sortedTranscriptConsequences', group_by_key='geneId')), - ], - options={ - 'db_table': 'GRCh38/MITO/annotations_memory', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), - }, - managers=[ - ('objects', django.db.models.manager.Manager()), - ('_overwrite_base_manager', django.db.models.manager.Manager()), - ], - ), migrations.CreateModel( name='EntriesMito', fields=[ diff --git a/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py b/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py index 7a7dc55061..59fc9b61ab 100644 --- a/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py +++ b/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py @@ -7,8 +7,6 @@ import django.db.models.deletion import django.db.models.manager -from settings import CLICKHOUSE_IN_MEMORY_DIR, CLICKHOUSE_DATA_DIR - class Migration(migrations.Migration): @@ -17,70 +15,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.CreateModel( - name='AnnotationsDiskSv', - fields=[ - ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), - ('xpos', clickhouse_backend.models.UInt64Field()), - ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), - ('pos', clickhouse_backend.models.UInt32Field()), - ('end', clickhouse_backend.models.UInt32Field()), - ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('position', clickhouse_backend.models.UInt32Field(null=True, blank=True))], db_column='rg37LocusEnd')), - ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), - ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), - ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), - ('algorithms', clickhouse_backend.models.StringField(low_cardinality=True)), - ('bothsides_support', clickhouse_backend.models.BoolField(db_column='bothsidesSupport')), - ('cpx_intervals', clickhouse_search.backend.fields.NestedField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('start', clickhouse_backend.models.UInt32Field()), ('end', clickhouse_backend.models.UInt32Field()), ('type', clickhouse_backend.models.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')]))], db_column='cpxIntervals')), - ('end_chrom', clickhouse_backend.models.Enum8Field(null=True, blank=True, db_column='endChrom', choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), - ('sv_source_detail', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')]))], db_column='svSourceDetail')), - ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), - ('sv_type_detail', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, 'INS_iDEL'), (2, 'INVdel'), (3, 'INVdup'), (4, 'ME'), (5, 'ME:ALU'), (6, 'ME:LINE1'), (7, 'ME:SVA'), (8, 'dDUP'), (9, 'dDUP_iDEL'), (10, 'delINV'), (11, 'delINVdel'), (12, 'delINVdup'), (13, 'dupINV'), (14, 'dupINVdel'), (15, 'dupINVdup')], db_column='svTypeDetail')), - ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), - ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_svs', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field()), ('id', clickhouse_backend.models.StringField())]))])), - ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), - ], - options={ - 'db_table': 'GRCh38/SV/annotations_disk', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), - }, - managers=[ - ('objects', django.db.models.manager.Manager()), - ('_overwrite_base_manager', django.db.models.manager.Manager()), - ], - ), - migrations.CreateModel( - name='AnnotationsSv', - fields=[ - ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), - ('xpos', clickhouse_backend.models.UInt64Field()), - ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), - ('pos', clickhouse_backend.models.UInt32Field()), - ('end', clickhouse_backend.models.UInt32Field()), - ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('position', clickhouse_backend.models.UInt32Field(null=True, blank=True))], db_column='rg37LocusEnd')), - ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), - ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), - ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), - ('algorithms', clickhouse_backend.models.StringField(low_cardinality=True)), - ('bothsides_support', clickhouse_backend.models.BoolField(db_column='bothsidesSupport')), - ('cpx_intervals', clickhouse_search.backend.fields.NestedField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('start', clickhouse_backend.models.UInt32Field()), ('end', clickhouse_backend.models.UInt32Field()), ('type', clickhouse_backend.models.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')]))], db_column='cpxIntervals')), - ('end_chrom', clickhouse_backend.models.Enum8Field(null=True, blank=True, db_column='endChrom', choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), - ('sv_source_detail', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')]))], db_column='svSourceDetail')), - ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), - ('sv_type_detail', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, 'INS_iDEL'), (2, 'INVdel'), (3, 'INVdup'), (4, 'ME'), (5, 'ME:ALU'), (6, 'ME:LINE1'), (7, 'ME:SVA'), (8, 'dDUP'), (9, 'dDUP_iDEL'), (10, 'delINV'), (11, 'delINVdel'), (12, 'delINVdup'), (13, 'dupINV'), (14, 'dupINVdel'), (15, 'dupINVdup')], db_column='svTypeDetail')), - ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), - ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_svs', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field()), ('id', clickhouse_backend.models.StringField())]))])), - ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), - ], - options={ - 'db_table': 'GRCh38/SV/annotations_memory', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), - }, - managers=[ - ('objects', django.db.models.manager.Manager()), - ('_overwrite_base_manager', django.db.models.manager.Manager()), - ], - ), migrations.CreateModel( name='EntriesSv', fields=[ diff --git a/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py b/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py index 31b25c4b02..dbc26fa75e 100644 --- a/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py +++ b/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py @@ -7,7 +7,6 @@ import django.db.models.deletion import django.db.models.manager -from settings import CLICKHOUSE_IN_MEMORY_DIR, CLICKHOUSE_DATA_DIR class Migration(migrations.Migration): @@ -16,60 +15,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.CreateModel( - name='AnnotationsDiskGcnv', - fields=[ - ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), - ('xpos', clickhouse_backend.models.UInt64Field()), - ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), - ('pos', clickhouse_backend.models.UInt32Field()), - ('end', clickhouse_backend.models.UInt32Field()), - ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('position', clickhouse_backend.models.UInt32Field(null=True, blank=True))], db_column='rg37LocusEnd')), - ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), - ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), - ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), - ('num_exon', clickhouse_backend.models.UInt16Field(db_column='numExon')), - ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), - ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), - ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sv_callset', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field())]))])), - ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), - ], - options={ - 'db_table': 'GRCh38/GCNV/annotations_disk', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), - }, - managers=[ - ('objects', django.db.models.manager.Manager()), - ('_overwrite_base_manager', django.db.models.manager.Manager()), - ], - ), - migrations.CreateModel( - name='AnnotationsGcnv', - fields=[ - ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), - ('xpos', clickhouse_backend.models.UInt64Field()), - ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), - ('pos', clickhouse_backend.models.UInt32Field()), - ('end', clickhouse_backend.models.UInt32Field()), - ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('position', clickhouse_backend.models.UInt32Field(null=True, blank=True))], db_column='rg37LocusEnd')), - ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), - ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), - ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), - ('num_exon', clickhouse_backend.models.UInt16Field(db_column='numExon')), - ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), - ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), - ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sv_callset', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field())]))])), - ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), - ], - options={ - 'db_table': 'GRCh38/GCNV/annotations_memory', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), - }, - managers=[ - ('objects', django.db.models.manager.Manager()), - ('_overwrite_base_manager', django.db.models.manager.Manager()), - ], - ), migrations.CreateModel( name='EntriesGcnv', fields=[ diff --git a/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py b/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py index d1fea85abc..b8ae51adc6 100644 --- a/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py +++ b/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py @@ -14,24 +14,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.DeleteModel( - name='AnnotationsDiskGcnv', - ), - migrations.DeleteModel( - name='AnnotationsDiskMito', - ), - migrations.DeleteModel( - name='AnnotationsDiskSv', - ), - migrations.DeleteModel( - name='AnnotationsGcnv', - ), - migrations.DeleteModel( - name='AnnotationsMito', - ), - migrations.DeleteModel( - name='AnnotationsSv', - ), migrations.CreateModel( name='AnnotationsMito', fields=[ From cb6a548137fe8466d17156410cdd337020bc286a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 Aug 2025 13:46:26 -0400 Subject: [PATCH 44/46] Revert "clean up migrations to prevent recreation of tables in empty db" This reverts commit f847b27bd60c3950956d6d043ff95c839b2f3890. --- ...to_annotationsmito_entriesmito_and_more.py | 54 +++++++++++++++ ...notationsdisksv_annotationssv_entriessv.py | 66 +++++++++++++++++++ ...nv_annotationsgcnv_entriesgcnv_and_more.py | 55 ++++++++++++++++ ...012_delete_annotationsdiskgcnv_and_more.py | 18 +++++ 4 files changed, 193 insertions(+) diff --git a/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py b/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py index 7cc402bffd..a88e7cbf64 100644 --- a/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py +++ b/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py @@ -7,6 +7,8 @@ import django.db.models.deletion import django.db.models.manager +from settings import CLICKHOUSE_IN_MEMORY_DIR, CLICKHOUSE_DATA_DIR + class Migration(migrations.Migration): @@ -15,6 +17,58 @@ class Migration(migrations.Migration): ] operations = [ + migrations.CreateModel( + name='AnnotationsDiskMito', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('pos', clickhouse_backend.models.UInt32Field()), + ('ref', clickhouse_backend.models.StringField()), + ('alt', clickhouse_backend.models.StringField()), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('rsid', clickhouse_backend.models.StringField(blank=True, null=True)), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('common_low_heteroplasmy', clickhouse_backend.models.BoolField(blank=True, db_column='commonLowHeteroplasmy', null=True)), + ('mitomap_pathogenic', clickhouse_backend.models.BoolField(blank=True, db_column='mitomapPathogenic', null=True)), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('apogee', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('haplogroup_defining', clickhouse_backend.models.BoolField(null=True, blank=True)), ('hmtvar', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mitotip', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'likely_pathogenic'), (1, 'possibly_pathogenic'), (2, 'possibly_benign'), (3, 'likely_benign')], null=True)), ('mut_taster', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'D'), (1, 'A'), (2, 'N'), (3, 'P')], null=True)), ('sift', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mlc', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_mito', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('gnomad_mito_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))])), ('helix', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('helix_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))]))])), + ('sorted_transcript_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('aminoAcids', clickhouse_backend.models.StringField(blank=True, null=True)), ('biotype', clickhouse_backend.models.StringField(blank=True, null=True)), ('canonical', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('codons', clickhouse_backend.models.StringField(blank=True, null=True)), ('consequenceTerms', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True))), ('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsc', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsp', clickhouse_backend.models.StringField(blank=True, null=True)), ('loftee', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('isLofNagnag', clickhouse_backend.models.BoolField(blank=True, null=True)), ('lofFilters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(blank=True, null=True)))])), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True)), ('transcriptId', clickhouse_backend.models.StringField()), ('transcriptRank', clickhouse_backend.models.UInt8Field())], db_column='sortedTranscriptConsequences', group_by_key='geneId')), + ], + options={ + 'db_table': 'GRCh38/MITO/annotations_disk', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsMito', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('pos', clickhouse_backend.models.UInt32Field()), + ('ref', clickhouse_backend.models.StringField()), + ('alt', clickhouse_backend.models.StringField()), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('rsid', clickhouse_backend.models.StringField(blank=True, null=True)), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('common_low_heteroplasmy', clickhouse_backend.models.BoolField(blank=True, db_column='commonLowHeteroplasmy', null=True)), + ('mitomap_pathogenic', clickhouse_backend.models.BoolField(blank=True, db_column='mitomapPathogenic', null=True)), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('apogee', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('haplogroup_defining', clickhouse_backend.models.BoolField(null=True, blank=True)), ('hmtvar', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mitotip', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'likely_pathogenic'), (1, 'possibly_pathogenic'), (2, 'possibly_benign'), (3, 'likely_benign')], null=True)), ('mut_taster', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'D'), (1, 'A'), (2, 'N'), (3, 'P')], null=True)), ('sift', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mlc', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_mito', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('gnomad_mito_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))])), ('helix', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('helix_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))]))])), + ('sorted_transcript_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('aminoAcids', clickhouse_backend.models.StringField(blank=True, null=True)), ('biotype', clickhouse_backend.models.StringField(blank=True, null=True)), ('canonical', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('codons', clickhouse_backend.models.StringField(blank=True, null=True)), ('consequenceTerms', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True))), ('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsc', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsp', clickhouse_backend.models.StringField(blank=True, null=True)), ('loftee', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('isLofNagnag', clickhouse_backend.models.BoolField(blank=True, null=True)), ('lofFilters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(blank=True, null=True)))])), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True)), ('transcriptId', clickhouse_backend.models.StringField()), ('transcriptRank', clickhouse_backend.models.UInt8Field())], db_column='sortedTranscriptConsequences', group_by_key='geneId')), + ], + options={ + 'db_table': 'GRCh38/MITO/annotations_memory', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), migrations.CreateModel( name='EntriesMito', fields=[ diff --git a/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py b/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py index 59fc9b61ab..7a7dc55061 100644 --- a/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py +++ b/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py @@ -7,6 +7,8 @@ import django.db.models.deletion import django.db.models.manager +from settings import CLICKHOUSE_IN_MEMORY_DIR, CLICKHOUSE_DATA_DIR + class Migration(migrations.Migration): @@ -15,6 +17,70 @@ class Migration(migrations.Migration): ] operations = [ + migrations.CreateModel( + name='AnnotationsDiskSv', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('pos', clickhouse_backend.models.UInt32Field()), + ('end', clickhouse_backend.models.UInt32Field()), + ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('position', clickhouse_backend.models.UInt32Field(null=True, blank=True))], db_column='rg37LocusEnd')), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('algorithms', clickhouse_backend.models.StringField(low_cardinality=True)), + ('bothsides_support', clickhouse_backend.models.BoolField(db_column='bothsidesSupport')), + ('cpx_intervals', clickhouse_search.backend.fields.NestedField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('start', clickhouse_backend.models.UInt32Field()), ('end', clickhouse_backend.models.UInt32Field()), ('type', clickhouse_backend.models.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')]))], db_column='cpxIntervals')), + ('end_chrom', clickhouse_backend.models.Enum8Field(null=True, blank=True, db_column='endChrom', choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('sv_source_detail', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')]))], db_column='svSourceDetail')), + ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), + ('sv_type_detail', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, 'INS_iDEL'), (2, 'INVdel'), (3, 'INVdup'), (4, 'ME'), (5, 'ME:ALU'), (6, 'ME:LINE1'), (7, 'ME:SVA'), (8, 'dDUP'), (9, 'dDUP_iDEL'), (10, 'delINV'), (11, 'delINVdel'), (12, 'delINVdup'), (13, 'dupINV'), (14, 'dupINVdel'), (15, 'dupINVdup')], db_column='svTypeDetail')), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_svs', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field()), ('id', clickhouse_backend.models.StringField())]))])), + ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), + ], + options={ + 'db_table': 'GRCh38/SV/annotations_disk', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsSv', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('pos', clickhouse_backend.models.UInt32Field()), + ('end', clickhouse_backend.models.UInt32Field()), + ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('position', clickhouse_backend.models.UInt32Field(null=True, blank=True))], db_column='rg37LocusEnd')), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('algorithms', clickhouse_backend.models.StringField(low_cardinality=True)), + ('bothsides_support', clickhouse_backend.models.BoolField(db_column='bothsidesSupport')), + ('cpx_intervals', clickhouse_search.backend.fields.NestedField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('start', clickhouse_backend.models.UInt32Field()), ('end', clickhouse_backend.models.UInt32Field()), ('type', clickhouse_backend.models.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')]))], db_column='cpxIntervals')), + ('end_chrom', clickhouse_backend.models.Enum8Field(null=True, blank=True, db_column='endChrom', choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('sv_source_detail', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')]))], db_column='svSourceDetail')), + ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), + ('sv_type_detail', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, 'INS_iDEL'), (2, 'INVdel'), (3, 'INVdup'), (4, 'ME'), (5, 'ME:ALU'), (6, 'ME:LINE1'), (7, 'ME:SVA'), (8, 'dDUP'), (9, 'dDUP_iDEL'), (10, 'delINV'), (11, 'delINVdel'), (12, 'delINVdup'), (13, 'dupINV'), (14, 'dupINVdel'), (15, 'dupINVdup')], db_column='svTypeDetail')), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_svs', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field()), ('id', clickhouse_backend.models.StringField())]))])), + ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), + ], + options={ + 'db_table': 'GRCh38/SV/annotations_memory', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), migrations.CreateModel( name='EntriesSv', fields=[ diff --git a/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py b/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py index dbc26fa75e..31b25c4b02 100644 --- a/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py +++ b/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py @@ -7,6 +7,7 @@ import django.db.models.deletion import django.db.models.manager +from settings import CLICKHOUSE_IN_MEMORY_DIR, CLICKHOUSE_DATA_DIR class Migration(migrations.Migration): @@ -15,6 +16,60 @@ class Migration(migrations.Migration): ] operations = [ + migrations.CreateModel( + name='AnnotationsDiskGcnv', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('pos', clickhouse_backend.models.UInt32Field()), + ('end', clickhouse_backend.models.UInt32Field()), + ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('position', clickhouse_backend.models.UInt32Field(null=True, blank=True))], db_column='rg37LocusEnd')), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('num_exon', clickhouse_backend.models.UInt16Field(db_column='numExon')), + ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sv_callset', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field())]))])), + ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), + ], + options={ + 'db_table': 'GRCh38/GCNV/annotations_disk', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsGcnv', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('pos', clickhouse_backend.models.UInt32Field()), + ('end', clickhouse_backend.models.UInt32Field()), + ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(null=True, blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('position', clickhouse_backend.models.UInt32Field(null=True, blank=True))], db_column='rg37LocusEnd')), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('num_exon', clickhouse_backend.models.UInt16Field(db_column='numExon')), + ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sv_callset', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field())]))])), + ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), + ], + options={ + 'db_table': 'GRCh38/GCNV/annotations_memory', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), migrations.CreateModel( name='EntriesGcnv', fields=[ diff --git a/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py b/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py index b8ae51adc6..d1fea85abc 100644 --- a/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py +++ b/clickhouse_search/migrations/0012_delete_annotationsdiskgcnv_and_more.py @@ -14,6 +14,24 @@ class Migration(migrations.Migration): ] operations = [ + migrations.DeleteModel( + name='AnnotationsDiskGcnv', + ), + migrations.DeleteModel( + name='AnnotationsDiskMito', + ), + migrations.DeleteModel( + name='AnnotationsDiskSv', + ), + migrations.DeleteModel( + name='AnnotationsGcnv', + ), + migrations.DeleteModel( + name='AnnotationsMito', + ), + migrations.DeleteModel( + name='AnnotationsSv', + ), migrations.CreateModel( name='AnnotationsMito', fields=[ From df7daf7336eaa1014c42ee6c466b8762e8bc8cf9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 Aug 2025 13:47:37 -0400 Subject: [PATCH 45/46] create deprecated annotations tsables in fdifferent dir --- ...nnotationsdiskmito_annotationsmito_entriesmito_and_more.py | 4 ++-- .../0004_annotationsdisksv_annotationssv_entriessv.py | 4 ++-- ...nnotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py b/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py index a88e7cbf64..71a170748f 100644 --- a/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py +++ b/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py @@ -36,7 +36,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/MITO/annotations_disk', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/tmp/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), @@ -62,7 +62,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/MITO/annotations_memory', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/tmp/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), diff --git a/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py b/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py index 7a7dc55061..62d1db667d 100644 --- a/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py +++ b/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py @@ -42,7 +42,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/SV/annotations_disk', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/tmp/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), @@ -74,7 +74,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/SV/annotations_memory', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/tmp/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), diff --git a/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py b/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py index 31b25c4b02..094c2c6453 100644 --- a/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py +++ b/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py @@ -36,7 +36,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/GCNV/annotations_disk', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/tmp/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), @@ -63,7 +63,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/GCNV/annotations_memory', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/tmp/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), From a530e21c217293f317deea8b97863193820e4440 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 Aug 2025 13:51:35 -0400 Subject: [PATCH 46/46] codacy --- ...nnotationsdiskmito_annotationsmito_entriesmito_and_more.py | 4 ++-- .../0004_annotationsdisksv_annotationssv_entriessv.py | 4 ++-- ...nnotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py b/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py index 71a170748f..630bb5016b 100644 --- a/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py +++ b/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py @@ -36,7 +36,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/MITO/annotations_disk', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/tmp/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/v0/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), @@ -62,7 +62,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/MITO/annotations_memory', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/tmp/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/v0/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), diff --git a/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py b/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py index 62d1db667d..b9dc348dd3 100644 --- a/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py +++ b/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py @@ -42,7 +42,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/SV/annotations_disk', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/tmp/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/v0/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), @@ -74,7 +74,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/SV/annotations_memory', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/tmp/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/v0/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), diff --git a/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py b/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py index 094c2c6453..cb3ca027d4 100644 --- a/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py +++ b/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py @@ -36,7 +36,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/GCNV/annotations_disk', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/tmp/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/v0/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), @@ -63,7 +63,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/GCNV/annotations_memory', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/tmp/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/v0/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()),