From 0ca602590a0d731efbbab938dd95010201071176 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 14:07:12 -0400 Subject: [PATCH 001/173] update test requirements --- vlm/requirements-test.in | 4 ++-- vlm/requirements-test.txt | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/vlm/requirements-test.in b/vlm/requirements-test.in index c63f835ebe..ef65b77d48 100644 --- a/vlm/requirements-test.in +++ b/vlm/requirements-test.in @@ -1,4 +1,4 @@ aioresponses coverage<5.2 -pytest-aiohttp -clickhouse-connect==0.8.18 +Django>=4.2,<4.3 +pytest-aiohttp \ No newline at end of file diff --git a/vlm/requirements-test.txt b/vlm/requirements-test.txt index ccd84ac4f7..a55807947e 100644 --- a/vlm/requirements-test.txt +++ b/vlm/requirements-test.txt @@ -14,14 +14,16 @@ aioresponses==0.7.8 # via -r vlm/requirements-test.in aiosignal==1.3.1 # via aiohttp +asgiref==3.6.0 + # via django async-timeout==4.0.2 # via aiohttp attrs==23.1.0 # via aiohttp -clickhouse-connect==0.8.18 - # via -r vlm/requirements-test.in coverage==5.1 # via -r vlm/requirements-test.in +django==4.2.27 + # via -r vlm/requirements-test.in exceptiongroup==1.1.3 # via pytest frozenlist==1.3.3 @@ -52,6 +54,8 @@ pytest-aiohttp==1.0.4 # via -r vlm/requirements-test.in pytest-asyncio==0.21.0 # via pytest-aiohttp +sqlparse==0.5.4 + # via django tomli==2.0.1 # via pytest yarl==1.17.2 From b95b6756324fdb96d0a956655393f4d7eaf31717 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 14:17:12 -0400 Subject: [PATCH 002/173] use django for db setup --- .github/workflows/vlm-unit-tests.yaml | 14 +++++++++++- vlm/setup_clickhouse_test_data.py | 31 --------------------------- 2 files changed, 13 insertions(+), 32 deletions(-) delete mode 100644 vlm/setup_clickhouse_test_data.py diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 2199726d0e..46093ab148 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -46,7 +46,19 @@ jobs: python3 -m pip install --upgrade pip wheel pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Settings and Data - run: python3 vlm/setup_clickhouse_test_data.py clickhouse 8123 clickhouse_test_user clickhouse_test_password + run: | + export CLICKHOUSE_WRITER_USER=clickhouse_test_user + export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password + export CLICKHOUSE_SERVICE_HOSTNAME=localhost + export CLICKHOUSE_SERVICE_PORT=8123 + python3 -u manage.py migrate --database=clickhouse_write + python3 -u manage.py loaddata clickhouse_search + docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" + docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - name: Run coverage tests run: | export SEQR_BASE_URL=https://test-seqr.org/ diff --git a/vlm/setup_clickhouse_test_data.py b/vlm/setup_clickhouse_test_data.py deleted file mode 100644 index a50334040c..0000000000 --- a/vlm/setup_clickhouse_test_data.py +++ /dev/null @@ -1,31 +0,0 @@ -import clickhouse_connect -import sys - -def setup_clickhouse_test_data(host, port, username, password): - client = clickhouse_connect.get_client(host=host, port=port, username=username, password=password) - client.command('CREATE DATABASE test_seqr') - - client.command('CREATE TABLE test_seqr.`GRCh37/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId') - client.insert('GRCh37/SNV_INDEL/key_lookup', data=[['7-143270172-A-G', 1], ['1-39190091-T-G', 2]], database='test_seqr') - client.command( - 'CREATE DICTIONARY test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` (key UInt32, ac_wes UInt32, ac_wgs UInt32, hom_wes UInt32, hom_wgs UInt32) PRIMARY KEY key SOURCE(CLICKHOUSE(USER %s PASSWORD %s QUERY "SELECT * FROM VALUES ((1, 4104, 607, 1276, 232), (2, 7, 2, 2, 1))")) LIFETIME(0) LAYOUT(FLAT(MAX_ARRAY_SIZE 500000000))', - parameters=(username, password), - ) - - client.command('CREATE TABLE test_seqr.`GRCh38/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId') - client.insert('GRCh38/SNV_INDEL/key_lookup', data=[['1-38724419-T-G', 1]], database='test_seqr') - client.command( - 'CREATE DICTIONARY test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` (key UInt32, ac_wes UInt32, ac_wgs UInt32, hom_wes UInt32, hom_wgs UInt32) PRIMARY KEY key SOURCE(CLICKHOUSE(USER %s PASSWORD %s QUERY "SELECT * FROM VALUES ((1, 18, 10, 3, 1))")) LIFETIME(0) LAYOUT(FLAT(MAX_ARRAY_SIZE 500000000))', - parameters=(username, password), - ) - - client.command("CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'") - client.command('GRANT SELECT ON test_seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user') - client.command('GRANT SELECT ON test_seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user') - client.command('GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user') - client.command('GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user') - - -if __name__ == '__main__': - args = sys.argv[1:] - setup_clickhouse_test_data(*args) From fe4dd3422ca86550e0bc561eef30c19c33d3570c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 14:28:05 -0400 Subject: [PATCH 003/173] update requiremnts --- vlm/requirements-test.in | 6 ++++-- vlm/requirements-test.txt | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/vlm/requirements-test.in b/vlm/requirements-test.in index ef65b77d48..b399d225ac 100644 --- a/vlm/requirements-test.in +++ b/vlm/requirements-test.in @@ -1,4 +1,6 @@ aioresponses coverage<5.2 -Django>=4.2,<4.3 -pytest-aiohttp \ No newline at end of file +django>=4.2,<4.3 +django-clickhouse-backend +django-guardian +pytest-aiohttp diff --git a/vlm/requirements-test.txt b/vlm/requirements-test.txt index a55807947e..742e2880e4 100644 --- a/vlm/requirements-test.txt +++ b/vlm/requirements-test.txt @@ -20,9 +20,18 @@ async-timeout==4.0.2 # via aiohttp attrs==23.1.0 # via aiohttp +clickhouse-driver==0.2.9 + # via django-clickhouse-backend coverage==5.1 # via -r vlm/requirements-test.in django==4.2.27 + # via + # -r vlm/requirements-test.in + # django-clickhouse-backend + # django-guardian +django-clickhouse-backend==1.3.2 + # via -r vlm/requirements-test.in +django-guardian==2.4.0 # via -r vlm/requirements-test.in exceptiongroup==1.1.3 # via pytest @@ -54,9 +63,13 @@ pytest-aiohttp==1.0.4 # via -r vlm/requirements-test.in pytest-asyncio==0.21.0 # via pytest-aiohttp +pytz==2022.7.1 + # via clickhouse-driver sqlparse==0.5.4 # via django tomli==2.0.1 # via pytest +tzlocal==5.3.1 + # via clickhouse-driver yarl==1.17.2 # via aiohttp From 068833feb71b81781ea1251a0ccac5d5a81feab0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 14:31:58 -0400 Subject: [PATCH 004/173] use seqr requiremnts --- .github/workflows/vlm-unit-tests.yaml | 1 + vlm/requirements-test.in | 3 --- vlm/requirements-test.txt | 19 ------------------- 3 files changed, 1 insertion(+), 22 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 46093ab148..8267baa51d 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -44,6 +44,7 @@ jobs: - name: Install dependencies run: | python3 -m pip install --upgrade pip wheel + pip install -r requirements.txt pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Settings and Data run: | diff --git a/vlm/requirements-test.in b/vlm/requirements-test.in index b399d225ac..f335034028 100644 --- a/vlm/requirements-test.in +++ b/vlm/requirements-test.in @@ -1,6 +1,3 @@ aioresponses coverage<5.2 -django>=4.2,<4.3 -django-clickhouse-backend -django-guardian pytest-aiohttp diff --git a/vlm/requirements-test.txt b/vlm/requirements-test.txt index 742e2880e4..df2cb3b32d 100644 --- a/vlm/requirements-test.txt +++ b/vlm/requirements-test.txt @@ -14,25 +14,12 @@ aioresponses==0.7.8 # via -r vlm/requirements-test.in aiosignal==1.3.1 # via aiohttp -asgiref==3.6.0 - # via django async-timeout==4.0.2 # via aiohttp attrs==23.1.0 # via aiohttp -clickhouse-driver==0.2.9 - # via django-clickhouse-backend coverage==5.1 # via -r vlm/requirements-test.in -django==4.2.27 - # via - # -r vlm/requirements-test.in - # django-clickhouse-backend - # django-guardian -django-clickhouse-backend==1.3.2 - # via -r vlm/requirements-test.in -django-guardian==2.4.0 - # via -r vlm/requirements-test.in exceptiongroup==1.1.3 # via pytest frozenlist==1.3.3 @@ -63,13 +50,7 @@ pytest-aiohttp==1.0.4 # via -r vlm/requirements-test.in pytest-asyncio==0.21.0 # via pytest-aiohttp -pytz==2022.7.1 - # via clickhouse-driver -sqlparse==0.5.4 - # via django tomli==2.0.1 # via pytest -tzlocal==5.3.1 - # via clickhouse-driver yarl==1.17.2 # via aiohttp From 809a6d3a1968060b89954f548f6c8df2dfeb8d6e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 14:35:00 -0400 Subject: [PATCH 005/173] use venv --- .github/workflows/vlm-unit-tests.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 8267baa51d..b58ff8697e 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -44,10 +44,12 @@ jobs: - name: Install dependencies run: | python3 -m pip install --upgrade pip wheel - pip install -r requirements.txt pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Settings and Data run: | + python3 -m venv venv + source venv/bin/activate + pip install -r requirements.txt export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost From 261cdd00fb101af08a1e911a1a2680f62a558b3f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 14:36:40 -0400 Subject: [PATCH 006/173] no seqr app migrations --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index b58ff8697e..6bc3019943 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -54,7 +54,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export CLICKHOUSE_SERVICE_PORT=8123 - python3 -u manage.py migrate --database=clickhouse_write + python3 -u manage.py migrate clickhouse_search --database=clickhouse_write python3 -u manage.py loaddata clickhouse_search docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" From f115f207ffb2d5b813cc0fb5cf26ebebb8fc81b1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 14:42:00 -0400 Subject: [PATCH 007/173] fix venv --- .github/workflows/vlm-unit-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 6bc3019943..6f43a292e5 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -47,8 +47,8 @@ jobs: pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Settings and Data run: | - python3 -m venv venv - source venv/bin/activate + python3 -m venv .venv + source .venv/bin/activate pip install -r requirements.txt export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password From 62eec9a3cb6a029da0453d7a339fae4496a4a98c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 14:43:17 -0400 Subject: [PATCH 008/173] split setup steps --- .github/workflows/vlm-unit-tests.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 6f43a292e5..c4d45d1d29 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -45,7 +45,7 @@ jobs: run: | python3 -m pip install --upgrade pip wheel pip install -r vlm/requirements-test.txt - - name: Set up Clickhouse Settings and Data + - name: Set up Clickhouse Database run: | python3 -m venv .venv source .venv/bin/activate @@ -56,6 +56,8 @@ jobs: export CLICKHOUSE_SERVICE_PORT=8123 python3 -u manage.py migrate clickhouse_search --database=clickhouse_write python3 -u manage.py loaddata clickhouse_search + - name: Set up Clickhouse Settings + run: | docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" From 13d81762a4efc04946491f9a75db77d9f34a6fe8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 14:43:36 -0400 Subject: [PATCH 009/173] remove venv for now --- .github/workflows/vlm-unit-tests.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index c4d45d1d29..296d6ec9d0 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -47,8 +47,6 @@ jobs: pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Database run: | - python3 -m venv .venv - source .venv/bin/activate pip install -r requirements.txt export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password From 74fa807189d7ca40a0efe95e476319385605290a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 20 May 2026 14:48:13 -0400 Subject: [PATCH 010/173] add postgres --- .github/workflows/vlm-unit-tests.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 296d6ec9d0..a0d07bf2cf 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -38,6 +38,17 @@ jobs: CLICKHOUSE_USER: clickhouse_test_user CLICKHOUSE_PASSWORD: clickhouse_test_password ALLOW_EMPTY_PASSWORD: no + postgres: + image: postgres + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + POSTGRES_PASSWORD: pgtest steps: - uses: actions/checkout@v2 @@ -46,8 +57,8 @@ jobs: python3 -m pip install --upgrade pip wheel pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Database - run: | - pip install -r requirements.txt + run: | # TODO add venv + pip install -r requirements.txt export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost @@ -61,6 +72,7 @@ jobs: docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - name: Run coverage tests run: | From a60662472b9c8f028d69e5cd97c607745c9a535e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 10:26:41 -0400 Subject: [PATCH 011/173] depndency install order --- .github/workflows/vlm-unit-tests.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index bf2f4a0b0a..fb539dd831 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -52,11 +52,8 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip wheel - pip install -r vlm/requirements.txt - pip install -r vlm/requirements-test.txt + - name: Install pip + run: python3 -m pip install --upgrade pip wheel - name: Set up Clickhouse Database run: | # TODO add venv pip install -r requirements.txt @@ -75,6 +72,10 @@ jobs: docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" + - name: Install dependencies + run: | + pip install -r vlm/requirements.txt + pip install -r vlm/requirements-test.txt - name: Run coverage tests run: | export SEQR_BASE_URL=https://test-seqr.org/ From 79f1531acf2df06cf093f0aed896a013529b9d40 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 10:31:05 -0400 Subject: [PATCH 012/173] psycopg setup --- .github/workflows/vlm-unit-tests.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index fb539dd831..403a9175f6 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -55,7 +55,8 @@ jobs: - name: Install pip run: python3 -m pip install --upgrade pip wheel - name: Set up Clickhouse Database - run: | # TODO add venv + run: | + pip install psycopg-binary pip install -r requirements.txt export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password @@ -74,7 +75,7 @@ jobs: docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - name: Install dependencies run: | - pip install -r vlm/requirements.txt + pip install -r vlm/requirements.txt # TODO clean up other packages pip install -r vlm/requirements-test.txt - name: Run coverage tests run: | From d2c1aaaa84c64ee830b21369f73f373839acd6e7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 10:33:40 -0400 Subject: [PATCH 013/173] add clickhous eports --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 403a9175f6..bfc61ca439 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -28,6 +28,7 @@ jobs: image: bitnamilegacy/clickhouse:latest ports: - 8123:8123 # HTTP interface + - 9000:9000 # Native client interface options: >- --health-cmd "clickhouse-client --query 'SELECT 1'" --health-interval 10s @@ -61,7 +62,6 @@ jobs: export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost - export CLICKHOUSE_SERVICE_PORT=8123 python3 -u manage.py migrate clickhouse_search --database=clickhouse_write python3 -u manage.py loaddata clickhouse_search - name: Set up Clickhouse Settings From 73813236e7b80780ad99ce052af2349b4c58c665 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 10:34:10 -0400 Subject: [PATCH 014/173] env reset --- .github/workflows/vlm-unit-tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index bfc61ca439..c4efe5ae2b 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -75,7 +75,8 @@ jobs: docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - name: Install dependencies run: | - pip install -r vlm/requirements.txt # TODO clean up other packages + pip freeze | xargs pip uninstall -y + pip install -r vlm/requirements.txt pip install -r vlm/requirements-test.txt - name: Run coverage tests run: | From 203aa1edd5df862cbd96f2cb35c70c8150499927 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 10:45:37 -0400 Subject: [PATCH 015/173] fix clcikhouse host --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index c4efe5ae2b..54474dacdb 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -61,7 +61,7 @@ jobs: pip install -r requirements.txt export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password - export CLICKHOUSE_SERVICE_HOSTNAME=localhost + export CLICKHOUSE_SERVICE_HOSTNAME=clickhouse python3 -u manage.py migrate clickhouse_search --database=clickhouse_write python3 -u manage.py loaddata clickhouse_search - name: Set up Clickhouse Settings From 8bd7a87043a7232a03c5b39d5cf845262bfc6969 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 10:48:22 -0400 Subject: [PATCH 016/173] explicitly create database --- .github/workflows/vlm-unit-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 54474dacdb..296bb4bdb4 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -62,6 +62,7 @@ jobs: export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=clickhouse + docker exec clickhouse clickhouse-client --query "CREATE DATABASE seqr" python3 -u manage.py migrate clickhouse_search --database=clickhouse_write python3 -u manage.py loaddata clickhouse_search - name: Set up Clickhouse Settings From a0f6eb6fba963687509673e86096e3810094a4eb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 10:53:34 -0400 Subject: [PATCH 017/173] use default github actions docker --- .github/workflows/vlm-unit-tests.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 296bb4bdb4..888b98078d 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -21,7 +21,6 @@ on: jobs: vlm_clickhouse: runs-on: ubuntu-latest - container: python:3.11-slim-bullseye services: clickhouse: @@ -52,6 +51,10 @@ jobs: POSTGRES_PASSWORD: pgtest steps: + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.11 - uses: actions/checkout@v2 - name: Install pip run: python3 -m pip install --upgrade pip wheel From 81766d7934fe3c72743b17f786c7905b822cb993 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 10:56:03 -0400 Subject: [PATCH 018/173] fix clickhouse hostname --- .github/workflows/vlm-unit-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 888b98078d..c17598123f 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -64,7 +64,7 @@ jobs: pip install -r requirements.txt export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password - export CLICKHOUSE_SERVICE_HOSTNAME=clickhouse + export CLICKHOUSE_SERVICE_HOSTNAME=localhost docker exec clickhouse clickhouse-client --query "CREATE DATABASE seqr" python3 -u manage.py migrate clickhouse_search --database=clickhouse_write python3 -u manage.py loaddata clickhouse_search @@ -86,7 +86,7 @@ jobs: run: | export SEQR_BASE_URL=https://test-seqr.org/ export NODE_ID=TestVLM - export CLICKHOUSE_SERVICE_HOSTNAME=clickhouse + export CLICKHOUSE_SERVICE_HOSTNAME=localhost export CLICKHOUSE_SERVICE_PORT=8123 export CLICKHOUSE_VLM_USERNAME=vlm_test_user export CLICKHOUSE_VLM_PASSWORD=vlm_test_password From 2601e0cb9eac5f4f1a5b3165e285635fee132ccf Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 10:56:30 -0400 Subject: [PATCH 019/173] remove extra psycopg --- .github/workflows/vlm-unit-tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index c17598123f..59e8d935aa 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -60,7 +60,6 @@ jobs: run: python3 -m pip install --upgrade pip wheel - name: Set up Clickhouse Database run: | - pip install psycopg-binary pip install -r requirements.txt export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password From 4e68be23d8d1ab5fc7532e906a60c29e0d2531d2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 10:59:46 -0400 Subject: [PATCH 020/173] test setup order --- .github/workflows/vlm-unit-tests.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 59e8d935aa..ff3aacdcdc 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -58,17 +58,9 @@ jobs: - uses: actions/checkout@v2 - name: Install pip run: python3 -m pip install --upgrade pip wheel - - name: Set up Clickhouse Database - run: | - pip install -r requirements.txt - export CLICKHOUSE_WRITER_USER=clickhouse_test_user - export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password - export CLICKHOUSE_SERVICE_HOSTNAME=localhost - docker exec clickhouse clickhouse-client --query "CREATE DATABASE seqr" - python3 -u manage.py migrate clickhouse_search --database=clickhouse_write - python3 -u manage.py loaddata clickhouse_search - - name: Set up Clickhouse Settings + - name: Set up Clickhouse run: | + docker exec clickhouse clickhouse-client --query "CREATE DATABASE seqr" docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" @@ -76,6 +68,14 @@ jobs: docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" + - name: Set up Clickhouse Test Database + run: | + pip install -r requirements.txt + export CLICKHOUSE_WRITER_USER=clickhouse_test_user + export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password + export CLICKHOUSE_SERVICE_HOSTNAME=localhost + python3 -u manage.py migrate clickhouse_search --database=clickhouse_write + python3 -u manage.py loaddata clickhouse_search - name: Install dependencies run: | pip freeze | xargs pip uninstall -y From f4029bb03f48d640c23b5c690dd1015b524d1bd3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 11:00:03 -0400 Subject: [PATCH 021/173] add clikhouse data dirs --- .github/workflows/vlm-unit-tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index ff3aacdcdc..7382fcc323 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -28,6 +28,9 @@ jobs: ports: - 8123:8123 # HTTP interface - 9000:9000 # Native client interface + volumes: + - /var/tmp:/var/seqr/clickhouse-data + - /tmp:/in-memory-dir options: >- --health-cmd "clickhouse-client --query 'SELECT 1'" --health-interval 10s From 1781696f28f415b89b55c8922521e64fac2aa492 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 11:00:42 -0400 Subject: [PATCH 022/173] use venv --- .github/workflows/vlm-unit-tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 7382fcc323..c851e10581 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -73,6 +73,8 @@ jobs: docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - name: Set up Clickhouse Test Database run: | + python3 -m venv .venv + source .venv/bin/activate pip install -r requirements.txt export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password @@ -81,7 +83,6 @@ jobs: python3 -u manage.py loaddata clickhouse_search - name: Install dependencies run: | - pip freeze | xargs pip uninstall -y pip install -r vlm/requirements.txt pip install -r vlm/requirements-test.txt - name: Run coverage tests From 168372b8768bffd911cd01cf9988f3f2b72628e9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 11:02:43 -0400 Subject: [PATCH 023/173] reload users --- .github/workflows/vlm-unit-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index c851e10581..8161a30fcb 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -65,6 +65,7 @@ jobs: run: | docker exec clickhouse clickhouse-client --query "CREATE DATABASE seqr" docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" + docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user" From 9da4a709829e6ad48917f7c35f652cd9c10d2349 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 11:04:55 -0400 Subject: [PATCH 024/173] setup order --- .github/workflows/vlm-unit-tests.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 8161a30fcb..06cdea7cdb 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -61,19 +61,9 @@ jobs: - uses: actions/checkout@v2 - name: Install pip run: python3 -m pip install --upgrade pip wheel - - name: Set up Clickhouse - run: | - docker exec clickhouse clickhouse-client --query "CREATE DATABASE seqr" - docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" - docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" - docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - name: Set up Clickhouse Test Database run: | + docker exec clickhouse clickhouse-client --query "CREATE DATABASE seqr" python3 -m venv .venv source .venv/bin/activate pip install -r requirements.txt @@ -82,6 +72,16 @@ jobs: export CLICKHOUSE_SERVICE_HOSTNAME=localhost python3 -u manage.py migrate clickhouse_search --database=clickhouse_write python3 -u manage.py loaddata clickhouse_search + - name: Set up Clickhouse + run: | + docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" + docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" + docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" + docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - name: Install dependencies run: | pip install -r vlm/requirements.txt From 0c252710c9da4ca99e9e6e99c8ec8f7f29c5b954 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 11:08:48 -0400 Subject: [PATCH 025/173] run all db migrations --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 06cdea7cdb..590d79ba45 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -70,7 +70,7 @@ jobs: export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost - python3 -u manage.py migrate clickhouse_search --database=clickhouse_write + python3 -u manage.py migrate --database=clickhouse_write python3 -u manage.py loaddata clickhouse_search - name: Set up Clickhouse run: | From 9ee12e9ebc8ef60266acca7afbd622315c127fd4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 11:16:34 -0400 Subject: [PATCH 026/173] install postrgres db too --- .github/workflows/vlm-unit-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 590d79ba45..af7fce2f3f 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -70,6 +70,7 @@ jobs: export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost + python3 -u manage.py migrate python3 -u manage.py migrate --database=clickhouse_write python3 -u manage.py loaddata clickhouse_search - name: Set up Clickhouse From 17c64ac161cc6d4f649ae58e21872c17386c4ca9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 11:20:09 -0400 Subject: [PATCH 027/173] postgres db setup --- .github/workflows/vlm-unit-tests.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index af7fce2f3f..0fa85e1320 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -52,6 +52,7 @@ jobs: --health-retries 5 env: POSTGRES_PASSWORD: pgtest + POSTGRES_DB: seqrdb steps: - name: Set up Python @@ -70,10 +71,11 @@ jobs: export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost + export POSTGRES_SERVICE_HOSTNAME=localhost python3 -u manage.py migrate python3 -u manage.py migrate --database=clickhouse_write python3 -u manage.py loaddata clickhouse_search - - name: Set up Clickhouse + - name: Set up Clickhouse User run: | docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" From 0c8add8bf02dcb755dda78ebb48b00ab058a5396 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 11:36:38 -0400 Subject: [PATCH 028/173] use pytest for test db creation --- .github/workflows/vlm-unit-tests.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 0fa85e1320..1a93ab3c5c 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -52,7 +52,6 @@ jobs: --health-retries 5 env: POSTGRES_PASSWORD: pgtest - POSTGRES_DB: seqrdb steps: - name: Set up Python @@ -64,7 +63,6 @@ jobs: run: python3 -m pip install --upgrade pip wheel - name: Set up Clickhouse Test Database run: | - docker exec clickhouse clickhouse-client --query "CREATE DATABASE seqr" python3 -m venv .venv source .venv/bin/activate pip install -r requirements.txt @@ -72,9 +70,8 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -u manage.py migrate - python3 -u manage.py migrate --database=clickhouse_write - python3 -u manage.py loaddata clickhouse_search + pytest --create-db + python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" From cb28176d71e6e9b809b048a733b6c1d2bf7ebda9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 11:39:31 -0400 Subject: [PATCH 029/173] depndency install --- .github/workflows/vlm-unit-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 1a93ab3c5c..8f7b556760 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -65,6 +65,7 @@ jobs: run: | python3 -m venv .venv source .venv/bin/activate + pip install -r vlm/requirements-test.txt pip install -r requirements.txt export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password From dd8a174ab781c501ecbe81820ae06b9a7675dec4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 11:51:43 -0400 Subject: [PATCH 030/173] use setup_databases explicitly --- .github/workflows/vlm-unit-tests.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 8f7b556760..0cac5d0834 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -65,13 +65,12 @@ jobs: run: | python3 -m venv .venv source .venv/bin/activate - pip install -r vlm/requirements-test.txt pip install -r requirements.txt export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - pytest --create-db + python3 -c "from django.test.utils import setup_databases; setup_databases(verbosity=1, interactive=False)" python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From 2be770ccd58144124cb8c5dcfd4b716d3bf2666a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 11:54:23 -0400 Subject: [PATCH 031/173] pytest django --- .github/workflows/vlm-unit-tests.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 0cac5d0834..aa52c7e933 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -65,12 +65,14 @@ jobs: run: | python3 -m venv .venv source .venv/bin/activate + pip install -r vlm/requirements-test.txt + pip install pytest-django pip install -r requirements.txt export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -c "from django.test.utils import setup_databases; setup_databases(verbosity=1, interactive=False)" + pytest --create-db python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From 0849693ff76d0658a6c5be9dec4b31b2a0b21464 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 11:56:29 -0400 Subject: [PATCH 032/173] clean up --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index aa52c7e933..9249880fca 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -97,6 +97,6 @@ jobs: export CLICKHOUSE_VLM_USERNAME=vlm_test_user export CLICKHOUSE_VLM_PASSWORD=vlm_test_password export CLICKHOUSE_DATABASE=test_seqr - coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py" -m pytest vlm/ + coverage run --source="./vlm" --omit="./vlm/__main__.py" -m pytest vlm/ coverage report -m --fail-under=95 From 8d1c30096a2f14b531f973b598b305ff33191649 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 12:00:06 -0400 Subject: [PATCH 033/173] pytest setup runner --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 9249880fca..dc2800ec38 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -72,7 +72,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - pytest --create-db + python3 -m pytest --create-db python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From 7338a2c9d22591e5246de2048f7883c15df592e0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 12:06:40 -0400 Subject: [PATCH 034/173] debug --- .github/workflows/vlm-unit-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index dc2800ec38..3a8a05ec12 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -68,6 +68,7 @@ jobs: pip install -r vlm/requirements-test.txt pip install pytest-django pip install -r requirements.txt + which pytest export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost From 91c104f10d5fddc70851f683a9bb7e7abfffebce Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 12:09:48 -0400 Subject: [PATCH 035/173] undo debug --- .github/workflows/vlm-unit-tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 3a8a05ec12..dc2800ec38 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -68,7 +68,6 @@ jobs: pip install -r vlm/requirements-test.txt pip install pytest-django pip install -r requirements.txt - which pytest export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost From 964421be1d81af1a5d42b6205686c883581689cf Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 21 May 2026 12:10:45 -0400 Subject: [PATCH 036/173] do not actually run tests --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index dc2800ec38..bf51faf553 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -72,7 +72,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -m pytest --create-db + python3 -m pytest --create-db --collect-only python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From c0aef7b04c737f62ad68a4ca90f28a70233b8073 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 12:12:39 -0400 Subject: [PATCH 037/173] restart venv --- .github/workflows/vlm-unit-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index bf51faf553..b87a72210f 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -68,6 +68,7 @@ jobs: pip install -r vlm/requirements-test.txt pip install pytest-django pip install -r requirements.txt + deactivate && source .venv/bin/activate export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost From 8f43fd8e4aaf9b5bfc07fd93c73965b8013efc1d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 12:15:22 -0400 Subject: [PATCH 038/173] no venv debug --- .github/workflows/vlm-unit-tests.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index b87a72210f..e8c67be879 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -63,12 +63,9 @@ jobs: run: python3 -m pip install --upgrade pip wheel - name: Set up Clickhouse Test Database run: | - python3 -m venv .venv - source .venv/bin/activate pip install -r vlm/requirements-test.txt pip install pytest-django pip install -r requirements.txt - deactivate && source .venv/bin/activate export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost From 893750a416036b6d1fd8d69681849716ba53f9ca Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 12:18:42 -0400 Subject: [PATCH 039/173] correct db name --- .github/workflows/vlm-unit-tests.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index e8c67be879..0822525426 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -76,10 +76,10 @@ jobs: run: | docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT SELECT ON seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT dictGet ON seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - name: Install dependencies From 1d91c51808f164d42392de7733fde1f6c3ec5aaa Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 12:18:50 -0400 Subject: [PATCH 040/173] Revert "no venv debug" This reverts commit 8f43fd8e4aaf9b5bfc07fd93c73965b8013efc1d. --- .github/workflows/vlm-unit-tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 0822525426..cec6e6783f 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -63,9 +63,12 @@ jobs: run: python3 -m pip install --upgrade pip wheel - name: Set up Clickhouse Test Database run: | + python3 -m venv .venv + source .venv/bin/activate pip install -r vlm/requirements-test.txt pip install pytest-django pip install -r requirements.txt + deactivate && source .venv/bin/activate export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost From 8b9ede465df015acbb80e63bd2ed831a7563d153 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 12:19:15 -0400 Subject: [PATCH 041/173] install soiurce --- .github/workflows/vlm-unit-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index cec6e6783f..2665ea5539 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -65,6 +65,7 @@ jobs: run: | python3 -m venv .venv source .venv/bin/activate + pip install -e . pip install -r vlm/requirements-test.txt pip install pytest-django pip install -r requirements.txt From b96da893da16d06aeb23f0ab2443f521347335d5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 12:21:25 -0400 Subject: [PATCH 042/173] use explicit venv python for pytest --- .github/workflows/vlm-unit-tests.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 2665ea5539..5c79ff50d9 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -65,7 +65,6 @@ jobs: run: | python3 -m venv .venv source .venv/bin/activate - pip install -e . pip install -r vlm/requirements-test.txt pip install pytest-django pip install -r requirements.txt @@ -74,7 +73,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -m pytest --create-db --collect-only + .venv/bin/python3 -m pytest --create-db --collect-only python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From 1623ab8431de2769409c57462b8b325c9287cdfc Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 12:26:29 -0400 Subject: [PATCH 043/173] explicitly skip pytest ui --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 5c79ff50d9..e139561188 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -73,7 +73,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - .venv/bin/python3 -m pytest --create-db --collect-only + .venv/bin/python3 -m pytest --create-db --collect-only -k "not ui" python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From 288d763dc6fd8d25c5a0a251e9f06a52afed4e8c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 12:29:10 -0400 Subject: [PATCH 044/173] delete old jsx python test --- ui/create_jsx_test.py | 85 ------------------------------------------- 1 file changed, 85 deletions(-) delete mode 100644 ui/create_jsx_test.py diff --git a/ui/create_jsx_test.py b/ui/create_jsx_test.py deleted file mode 100644 index d6deeaf1f1..0000000000 --- a/ui/create_jsx_test.py +++ /dev/null @@ -1,85 +0,0 @@ -import argparse as ap -import re -import os - -p = ap.ArgumentParser("Create a .test.js file with a shallow-render test template for a .jsx component. " - "This script expects the .jsx file to export the class it defines using a statement like:" - "'export { SomeWidget as SomeWidgetComponent }'" ) - -p.add_argument('jsx_path', help='Path of .jsx component') -args = p.parse_args() - -jsx_dir = os.path.dirname(args.jsx_path) -jsx_filename = os.path.basename(args.jsx_path) - -outf = open(os.path.join(jsx_dir, jsx_filename.replace('.jsx', '.test.js')), 'w') - -class_name, component_name = None, None -propTypes = "" -mapStateToProps = "" -imports = "" -f = open(args.jsx_path) - -for line in f: - # match lines like 'export { FilterSelector as FilterSelectorComponent }' - match = re.match('export[ ]* \{[ ]*([^ ]+)[ ]* as [ ]*([^ ]+)[ ]*\}', line) - if match: - class_name = match.group(1) - component_name = match.group(2) - - match = re.match('class[ ]+([^ ]+)[ ]+extends[ ]+React.Component', line) - if match: - class_name = component_name = match.group(1) - - match = re.match('const[ ]+([^ ]+)[ ]+=[ (]+props[ )]+=>', line) - if match: - class_name = component_name = match.group(1) - - elif line.startswith('import') and 'rootReducer' in line: - imports += line - - elif "propTypes =" in line: - while '}' not in line: - line = next(f) - if ':' in line: - propTypes += ' ' + line.strip()+'\n' - propTypes = propTypes.rstrip('\n') - - elif "mapStateToProps =" in line: - while '}' not in line: - line = next(f) - if ':' in line: - mapStateToProps += ' ' + line.strip()+'\n' - mapStateToProps = mapStateToProps.replace('state', 'STATE1').rstrip('\n') - -if not class_name: - #p.error("export line (eg. 'export { FilterSelector as FilterSelectorComponent }') not found") - class_name = component_name = os.path.basename(args.jsx_path).replace('.jsx', '') - -if not mapStateToProps: - mapStateToProps = propTypes - - -template = """import React from 'react' -import { shallow, configure } from 'enzyme' -import Adapter from 'enzyme-adapter-react-16' -import %(component_name)s from './%(class_name)s' -%(imports)s - -configure({ adapter: new Adapter() }) - -test('shallow-render without crashing', () => { - /* -%(propTypes)s - */ - - const props = { -%(mapStateToProps)s - } - - shallow(<%(component_name)s {...props} />) -}) -""" % locals() - -outf.write(template) -outf.close() From c1a67fc3d36c2c6352f0ff92cf6cef15c2ff6558 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 12:32:10 -0400 Subject: [PATCH 045/173] clean up --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index e139561188..cec6e6783f 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -73,7 +73,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - .venv/bin/python3 -m pytest --create-db --collect-only -k "not ui" + python3 -m pytest --create-db --collect-only python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From d29c72567b43b72583e723a34546cad06a1da461 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 12:32:22 -0400 Subject: [PATCH 046/173] clean up --- .github/workflows/vlm-unit-tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index cec6e6783f..bf05b952f7 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -68,7 +68,6 @@ jobs: pip install -r vlm/requirements-test.txt pip install pytest-django pip install -r requirements.txt - deactivate && source .venv/bin/activate export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost From 9591a59a070f6e7da9c4af39cedd5856e0bee1ad Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 12:32:41 -0400 Subject: [PATCH 047/173] missing dependencies --- .github/workflows/vlm-unit-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index bf05b952f7..d954e625ab 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -68,6 +68,7 @@ jobs: pip install -r vlm/requirements-test.txt pip install pytest-django pip install -r requirements.txt + pip install -r requirements-dev.txt export CLICKHOUSE_WRITER_USER=clickhouse_test_user export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost From b208913540408871be61c70872359350aa4a56fe Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 14:03:22 -0400 Subject: [PATCH 048/173] more requirements --- .github/workflows/vlm-unit-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index d954e625ab..5c20395a23 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -65,6 +65,7 @@ jobs: run: | python3 -m venv .venv source .venv/bin/activate + pip install -r vlm/requirements.txt pip install -r vlm/requirements-test.txt pip install pytest-django pip install -r requirements.txt From b9d60dde6e8f7dc61f2370231d312200a62ae64a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 14:04:39 -0400 Subject: [PATCH 049/173] install order --- .github/workflows/vlm-unit-tests.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 5c20395a23..9607c79dff 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -59,8 +59,11 @@ jobs: with: python-version: 3.11 - uses: actions/checkout@v2 - - name: Install pip - run: python3 -m pip install --upgrade pip wheel + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip wheel + pip install -r vlm/requirements.txt + pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Test Database run: | python3 -m venv .venv @@ -86,10 +89,6 @@ jobs: docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - - name: Install dependencies - run: | - pip install -r vlm/requirements.txt - pip install -r vlm/requirements-test.txt - name: Run coverage tests run: | export SEQR_BASE_URL=https://test-seqr.org/ From 62dac69fb35e9167e3badc858b2ae90c33dd0008 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 14:06:44 -0400 Subject: [PATCH 050/173] DJANGO_SETTINGS_MODULE --- .github/workflows/vlm-unit-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 9607c79dff..0e3343a4cb 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -77,6 +77,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost + export DJANGO_SETTINGS_MODULE=settings python3 -m pytest --create-db --collect-only python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User From 8bbb867038f4b9a50154a6a9b1e3b4f0cc8bf401 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 14:24:30 -0400 Subject: [PATCH 051/173] use migrate directly --- .github/workflows/vlm-unit-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 0e3343a4cb..8c3b28be25 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -79,6 +79,7 @@ jobs: export POSTGRES_SERVICE_HOSTNAME=localhost export DJANGO_SETTINGS_MODULE=settings python3 -m pytest --create-db --collect-only + python3 -u manage.py migrate --database=clickhouse_write python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From 7880db343dd4c1562bc64f946e1f5b3e30b0f85c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 14:27:38 -0400 Subject: [PATCH 052/173] create db --- .github/workflows/vlm-unit-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 8c3b28be25..6c134a7d1c 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -66,6 +66,7 @@ jobs: pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Test Database run: | + docker exec clickhouse clickhouse-client --query "CREATE DATABASE seqr" python3 -m venv .venv source .venv/bin/activate pip install -r vlm/requirements.txt From 22378a6b37d1a053f3e0c75bd74d51c21fe44876 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 14:30:18 -0400 Subject: [PATCH 053/173] back to pytets --- .github/workflows/vlm-unit-tests.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 6c134a7d1c..5e348c5902 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -66,7 +66,6 @@ jobs: pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Test Database run: | - docker exec clickhouse clickhouse-client --query "CREATE DATABASE seqr" python3 -m venv .venv source .venv/bin/activate pip install -r vlm/requirements.txt @@ -78,9 +77,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - export DJANGO_SETTINGS_MODULE=settings - python3 -m pytest --create-db --collect-only - python3 -u manage.py migrate --database=clickhouse_write + python3 -m pytest --create-db --collect-only --ds=settings python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From aa9b3eceb3f896dcd61de9439c1799d80ed8b180 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 14:51:30 -0400 Subject: [PATCH 054/173] rename test db --- .github/workflows/vlm-unit-tests.yaml | 2 +- vlm/test_django_settings.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 vlm/test_django_settings.py diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 5e348c5902..0e1389d221 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -77,7 +77,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -m pytest --create-db --collect-only --ds=settings + python3 -m pytest --create-db --collect-only --ds=vlm.test_django_settings python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | diff --git a/vlm/test_django_settings.py b/vlm/test_django_settings.py new file mode 100644 index 0000000000..ed41568ade --- /dev/null +++ b/vlm/test_django_settings.py @@ -0,0 +1,6 @@ +from settings import * + +DATABASES = { + db_name: {**db, 'TEST': {**db.get('TEST', {}), 'NAME': db_name}} + for db_name, db in settings.DATABASES.items() +} \ No newline at end of file From 1f2f9281d8ea590937bd8f2994ac2dcefedeafb6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 14:54:03 -0400 Subject: [PATCH 055/173] fix syntax --- vlm/test_django_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vlm/test_django_settings.py b/vlm/test_django_settings.py index ed41568ade..42dc865d5a 100644 --- a/vlm/test_django_settings.py +++ b/vlm/test_django_settings.py @@ -2,5 +2,5 @@ DATABASES = { db_name: {**db, 'TEST': {**db.get('TEST', {}), 'NAME': db_name}} - for db_name, db in settings.DATABASES.items() + for db_name, db in DATABASES.items() } \ No newline at end of file From 794196dca2d534602e7d57a51a6fcfcc7a9cf4d5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 14:58:08 -0400 Subject: [PATCH 056/173] verbosity --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 0e1389d221..1996636795 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -77,7 +77,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -m pytest --create-db --collect-only --ds=vlm.test_django_settings + python3 -m pytest --create-db --collect-only --ds=vlm.test_django_settings -v python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From bc6030729d417d31ecb236c9273013bfe7b95de8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 14:59:26 -0400 Subject: [PATCH 057/173] actually keep db --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 1996636795..37e447d59e 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -77,7 +77,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -m pytest --create-db --collect-only --ds=vlm.test_django_settings -v + python3 -m pytest --create-db --reuse-db --collect-only --ds=vlm.test_django_settings -v python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From 74cdea0ebe3c7dc87040cdff0350cf47edebe86b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 15:06:26 -0400 Subject: [PATCH 058/173] more verbose --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 37e447d59e..38cb1acf67 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -77,7 +77,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -m pytest --create-db --reuse-db --collect-only --ds=vlm.test_django_settings -v + python3 -m pytest --create-db --reuse-db --collect-only --ds=vlm.test_django_settings -vv python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From 1ef8a206463aff1879bb1d621736746ce90d9e5c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 15:10:25 -0400 Subject: [PATCH 059/173] actually run test --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 38cb1acf67..686ca43529 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -77,7 +77,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -m pytest --create-db --reuse-db --collect-only --ds=vlm.test_django_settings -vv + python3 -m pytest --create-db --reuse-db --ds=vlm.test_django_settings -vv python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From 576247770d033b5817e43097c0bb635560d771ae Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 15:11:18 -0400 Subject: [PATCH 060/173] skip tests --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 686ca43529..1a0d4574bd 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -77,7 +77,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -m pytest --create-db --reuse-db --ds=vlm.test_django_settings -vv + python3 -m pytest --create-db --reuse-db -k "not vlm" --ds=vlm.test_django_settings -vv python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From 10d6ce3c76cf24cc48dd71150126bc17c6c573fc Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 15:13:49 -0400 Subject: [PATCH 061/173] run tests --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 1a0d4574bd..686ca43529 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -77,7 +77,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -m pytest --create-db --reuse-db -k "not vlm" --ds=vlm.test_django_settings -vv + python3 -m pytest --create-db --reuse-db --ds=vlm.test_django_settings -vv python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From 56a90123b42c56258d23d26d7c0c90e6466fc096 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 15:17:42 -0400 Subject: [PATCH 062/173] run single test --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 686ca43529..e83f5bd855 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -77,7 +77,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -m pytest --create-db --reuse-db --ds=vlm.test_django_settings -vv + python3 -m pytest vlm/test_vlm.py::VlmTestCase::test_status --create-db --reuse-db --ds=vlm.test_django_settings -vv python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From 52e3693cf08d181da50c76af05d66ebc24c1f35c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 15:18:24 -0400 Subject: [PATCH 063/173] do not recreate db --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index e83f5bd855..c322a67727 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -98,6 +98,6 @@ jobs: export CLICKHOUSE_VLM_USERNAME=vlm_test_user export CLICKHOUSE_VLM_PASSWORD=vlm_test_password export CLICKHOUSE_DATABASE=test_seqr - coverage run --source="./vlm" --omit="./vlm/__main__.py" -m pytest vlm/ + coverage run --source="./vlm" --omit="./vlm/__main__.py" -m pytest vlm/ --reuse-db coverage report -m --fail-under=95 From 0b684d284e9a4a048b93fabc5ddc28d7d87a7da4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 15:27:55 -0400 Subject: [PATCH 064/173] try setup only --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index c322a67727..a234dd0134 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -77,7 +77,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -m pytest vlm/test_vlm.py::VlmTestCase::test_status --create-db --reuse-db --ds=vlm.test_django_settings -vv + python3 -m pytest --setup-only --create-db --reuse-db --ds=vlm.test_django_settings -vv python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From b9a3952317c62b84e4f03e3c93d1612a9c298ea3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 16:30:22 -0400 Subject: [PATCH 065/173] unblock django db access --- vlm/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 vlm/conftest.py diff --git a/vlm/conftest.py b/vlm/conftest.py new file mode 100644 index 0000000000..828cc64c88 --- /dev/null +++ b/vlm/conftest.py @@ -0,0 +1,6 @@ +import pytest + +@pytest.fixture(autouse=True) +def enable_db_globally(django_db_blocker): + """Globally unblocks database access for all tests and fixtures.""" + django_db_blocker.unblock() From c1d5bdf8f2483b409442f2a10f58468cf39e119b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 16:42:33 -0400 Subject: [PATCH 066/173] enable databases for vlm tests --- vlm/test_vlm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vlm/test_vlm.py b/vlm/test_vlm.py index 02134c5e70..304ea7ffe8 100644 --- a/vlm/test_vlm.py +++ b/vlm/test_vlm.py @@ -9,6 +9,7 @@ REQUESTER_CLIENT_ID = 'abc123' class VlmTestCase(AioHTTPTestCase): + databases = '__all__' async def get_application(self): return await init_web_app() From d34aacee3c5d0089e0f7e57ef92fab14ea3b3013 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 16:51:42 -0400 Subject: [PATCH 067/173] better fixture --- vlm/conftest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/vlm/conftest.py b/vlm/conftest.py index 828cc64c88..7e0d44dd2e 100644 --- a/vlm/conftest.py +++ b/vlm/conftest.py @@ -1,6 +1,5 @@ import pytest @pytest.fixture(autouse=True) -def enable_db_globally(django_db_blocker): - """Globally unblocks database access for all tests and fixtures.""" - django_db_blocker.unblock() +def enable_db_access_for_all_tests(db): + pass From 5d60cc9dc2c473380e3096ce4c9f17c1e21dd690 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 17:08:50 -0400 Subject: [PATCH 068/173] custom db setup --- vlm/conftest.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/vlm/conftest.py b/vlm/conftest.py index 7e0d44dd2e..b780444175 100644 --- a/vlm/conftest.py +++ b/vlm/conftest.py @@ -3,3 +3,38 @@ @pytest.fixture(autouse=True) def enable_db_access_for_all_tests(db): pass + +@pytest.fixture(scope='session') +def django_db_setup( + request: pytest.FixtureRequest, + django_db_blocker: DjangoDbBlocker, + django_db_keepdb: bool, + django_db_createdb: bool, +): + """Top level fixture to ensure test databases are available""" + from django.test.utils import setup_databases, teardown_databases + + setup_databases_args = {} + + if django_db_keepdb and not django_db_createdb: + setup_databases_args["keepdb"] = True + + with django_db_blocker.unblock(): + db_cfg = setup_databases( + verbosity=request.config.option.verbose, + interactive=False, + aliases=['clickhouse_write'], + **setup_databases_args, + ) + # TODO loaddata + + yield + + if not django_db_keepdb: + with django_db_blocker.unblock(): + try: + teardown_databases(db_cfg, verbosity=request.config.option.verbose) + except Exception as exc: # noqa: BLE001 + request.node.warn( + pytest.PytestWarning(f"Error when trying to teardown test databases: {exc!r}") + ) \ No newline at end of file From e1564ff09bfb4c60c666b6aaf8002410bffcc1d3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 17:09:29 -0400 Subject: [PATCH 069/173] clean up --- vlm/test_vlm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vlm/test_vlm.py b/vlm/test_vlm.py index 304ea7ffe8..02134c5e70 100644 --- a/vlm/test_vlm.py +++ b/vlm/test_vlm.py @@ -9,7 +9,6 @@ REQUESTER_CLIENT_ID = 'abc123' class VlmTestCase(AioHTTPTestCase): - databases = '__all__' async def get_application(self): return await init_web_app() From d959a642079a6b9c36ca80418f977b774fda51ac Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 17:11:49 -0400 Subject: [PATCH 070/173] cleaner args --- vlm/conftest.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/vlm/conftest.py b/vlm/conftest.py index b780444175..61b0877434 100644 --- a/vlm/conftest.py +++ b/vlm/conftest.py @@ -9,22 +9,16 @@ def django_db_setup( request: pytest.FixtureRequest, django_db_blocker: DjangoDbBlocker, django_db_keepdb: bool, - django_db_createdb: bool, ): """Top level fixture to ensure test databases are available""" from django.test.utils import setup_databases, teardown_databases - setup_databases_args = {} - - if django_db_keepdb and not django_db_createdb: - setup_databases_args["keepdb"] = True - with django_db_blocker.unblock(): db_cfg = setup_databases( verbosity=request.config.option.verbose, interactive=False, aliases=['clickhouse_write'], - **setup_databases_args, + keepdb=django_db_keepdb, ) # TODO loaddata From f81607126b4d24ab7fa12c4f8e5a772e1ee2a379 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 17:12:40 -0400 Subject: [PATCH 071/173] fix typing --- vlm/conftest.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/vlm/conftest.py b/vlm/conftest.py index 61b0877434..276bd32b5b 100644 --- a/vlm/conftest.py +++ b/vlm/conftest.py @@ -5,12 +5,7 @@ def enable_db_access_for_all_tests(db): pass @pytest.fixture(scope='session') -def django_db_setup( - request: pytest.FixtureRequest, - django_db_blocker: DjangoDbBlocker, - django_db_keepdb: bool, -): - """Top level fixture to ensure test databases are available""" +def django_db_setup(request, django_db_blocker,django_db_keepdb): from django.test.utils import setup_databases, teardown_databases with django_db_blocker.unblock(): From 140ccdcbe7d6bd62b1ff80d8ce4f0b7439563a2e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 17:14:05 -0400 Subject: [PATCH 072/173] run vlm tests on fixture update --- .github/workflows/vlm-unit-tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index a234dd0134..9223d6bdda 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -12,11 +12,13 @@ on: paths: - 'vlm/**' - '.github/workflows/*vlm*.yaml' + - 'clickhouse_search/fixtures/clickhouse_search.json' pull_request: types: [opened, synchronize, reopened] paths: - 'vlm/**' - '.github/workflows/*vlm*.yaml' + - 'clickhouse_search/fixtures/clickhouse_search.json' jobs: vlm_clickhouse: From f8a979203c09dca7cb4b8a4667eb2fbd54d251d5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 17:15:41 -0400 Subject: [PATCH 073/173] no custom django settings --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 9223d6bdda..cd20c0bb34 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -79,7 +79,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -m pytest --setup-only --create-db --reuse-db --ds=vlm.test_django_settings -vv + python3 -m pytest --setup-only --create-db --reuse-db --ds=settings -vv python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse User run: | From fb5cf500334ac704005785db49c094945ebdc422 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 17:19:43 -0400 Subject: [PATCH 074/173] set up default db --- vlm/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vlm/conftest.py b/vlm/conftest.py index 276bd32b5b..e6d7a5220a 100644 --- a/vlm/conftest.py +++ b/vlm/conftest.py @@ -12,7 +12,7 @@ def django_db_setup(request, django_db_blocker,django_db_keepdb): db_cfg = setup_databases( verbosity=request.config.option.verbose, interactive=False, - aliases=['clickhouse_write'], + aliases=['default', 'clickhouse_write'], keepdb=django_db_keepdb, ) # TODO loaddata From 1dbeba80863ff406cf067c4c1030da3fb406791c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 17:38:13 -0400 Subject: [PATCH 075/173] clickhouse settings move up --- .github/workflows/vlm-unit-tests.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index cd20c0bb34..0eab9841b3 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -66,6 +66,19 @@ jobs: python3 -m pip install --upgrade pip wheel pip install -r vlm/requirements.txt pip install -r vlm/requirements-test.txt + - name: Set up Clickhouse Settings + run: | + cat < test_config.xml + + + + 1 + + + + EOF + docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml + docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" - name: Set up Clickhouse Test Database run: | python3 -m venv .venv @@ -89,7 +102,6 @@ jobs: docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - name: Run coverage tests run: | From a469c640f316efb32a1475ee8497575a76b6027d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 17:42:53 -0400 Subject: [PATCH 076/173] settings profile --- .github/workflows/vlm-unit-tests.yaml | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 0eab9841b3..a58dfb2201 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -68,17 +68,15 @@ jobs: pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Settings run: | - cat < test_config.xml - - - - 1 - - - - EOF - docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" + docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" + docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" + docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS allow_materialized_view_with_bad_select=1, stop_refreshable_materialized_views_on_startup=1 TO vlm_test_user, clickhouse_test_user" + docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - name: Set up Clickhouse Test Database run: | python3 -m venv .venv @@ -94,15 +92,6 @@ jobs: export POSTGRES_SERVICE_HOSTNAME=localhost python3 -m pytest --setup-only --create-db --reuse-db --ds=settings -vv python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - - name: Set up Clickhouse User - run: | - docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" - docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - name: Run coverage tests run: | export SEQR_BASE_URL=https://test-seqr.org/ From e1940030d2247d11107f265b3b9a2134a55ee157 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 17:46:43 -0400 Subject: [PATCH 077/173] clickhouse grant permissions --- .github/workflows/vlm-unit-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index a58dfb2201..b21c8c8cb7 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -68,15 +68,15 @@ jobs: pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Settings run: | - docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" + docker exec clickhouse clickhouse-client --query "GRANT CREATE NAMED COLLECTION TO clickhouse_test_user" docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" - docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS allow_materialized_view_with_bad_select=1, stop_refreshable_materialized_views_on_startup=1 TO vlm_test_user, clickhouse_test_user" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" + docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" - name: Set up Clickhouse Test Database run: | python3 -m venv .venv From b5f83cd9409ac2f1378dfbe29bc7acf7ad79ad0e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 26 May 2026 17:48:51 -0400 Subject: [PATCH 078/173] users config --- .github/workflows/vlm-unit-tests.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index b21c8c8cb7..7e59c15ba5 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -68,7 +68,16 @@ jobs: pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Settings run: | - docker exec clickhouse clickhouse-client --query "GRANT CREATE NAMED COLLECTION TO clickhouse_test_user" + cat < test_config.xml + + + + 1 + + + + EOF + docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" From 5a527478483fbf562ed33fa78f461391bb5c268e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 10:30:51 -0400 Subject: [PATCH 079/173] use config --- .github/workflows/vlm-unit-tests.yaml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 7e59c15ba5..9f6cf72ebe 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -74,17 +74,21 @@ jobs: 1 + + vlm_test_password + + GRANT SELECT ON test_seqr.`GRCh37/SNV_INDEL/key_lookup` + GRANT SELECT ON test_seqr.`GRCh38/SNV_INDEL/key_lookup` + GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict`/query> + GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict`/query> + + EOF docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml - docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" - docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS allow_materialized_view_with_bad_select=1, stop_refreshable_materialized_views_on_startup=1 TO vlm_test_user, clickhouse_test_user" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS allow_materialized_view_with_bad_select=1, stop_refreshable_materialized_views_on_startup=1 TO vlm_test_user, clickhouse_test_user" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" - name: Set up Clickhouse Test Database run: | From 6483e9e6d33bc58b18702d740f48f02949e98a2e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 10:31:16 -0400 Subject: [PATCH 080/173] clean up --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 9f6cf72ebe..63f837e78a 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -82,7 +82,7 @@ jobs: GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict`/query> GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict`/query> - + EOF From 984981c1a8480e62e05f9ba41a03d332439d85e0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 10:35:16 -0400 Subject: [PATCH 081/173] clean up --- .github/workflows/vlm-unit-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 63f837e78a..5dceb0e714 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -79,8 +79,8 @@ jobs: GRANT SELECT ON test_seqr.`GRCh37/SNV_INDEL/key_lookup` GRANT SELECT ON test_seqr.`GRCh38/SNV_INDEL/key_lookup` - GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict`/query> - GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict`/query> + GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` + GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` From ab63500021a057e776a5148c306f6921ecf78e92 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 10:38:20 -0400 Subject: [PATCH 082/173] remove user grants for now --- .github/workflows/vlm-unit-tests.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 5dceb0e714..65517e4f00 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -76,12 +76,6 @@ jobs: vlm_test_password - - GRANT SELECT ON test_seqr.`GRCh37/SNV_INDEL/key_lookup` - GRANT SELECT ON test_seqr.`GRCh38/SNV_INDEL/key_lookup` - GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` - GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` - From 99f715b20986b9c51522fa5dcb5edb7eb9ae454a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 10:43:13 -0400 Subject: [PATCH 083/173] reload users again --- .github/workflows/vlm-unit-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 65517e4f00..e0103c9341 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -84,6 +84,7 @@ jobs: docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS allow_materialized_view_with_bad_select=1, stop_refreshable_materialized_views_on_startup=1 TO vlm_test_user, clickhouse_test_user" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" + docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - name: Set up Clickhouse Test Database run: | python3 -m venv .venv From cc56f31dd56d5a7195043305305e93f391f37c4a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 10:44:19 -0400 Subject: [PATCH 084/173] readd user rants --- .github/workflows/vlm-unit-tests.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index e0103c9341..8842582a67 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -100,6 +100,13 @@ jobs: export POSTGRES_SERVICE_HOSTNAME=localhost python3 -m pytest --setup-only --create-db --reuse-db --ds=settings -vv python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write + - name: Set up Clickhouse test user + run: | + docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - name: Run coverage tests run: | export SEQR_BASE_URL=https://test-seqr.org/ From cfcee0423089efed5ced695c9a62a931f8a8c939 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 10:45:09 -0400 Subject: [PATCH 085/173] remove unused file --- vlm/test_django_settings.py | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 vlm/test_django_settings.py diff --git a/vlm/test_django_settings.py b/vlm/test_django_settings.py deleted file mode 100644 index 42dc865d5a..0000000000 --- a/vlm/test_django_settings.py +++ /dev/null @@ -1,6 +0,0 @@ -from settings import * - -DATABASES = { - db_name: {**db, 'TEST': {**db.get('TEST', {}), 'NAME': db_name}} - for db_name, db in DATABASES.items() -} \ No newline at end of file From a6f28f8f0f523d247be5f9d7472104b687ac03df Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 11:07:42 -0400 Subject: [PATCH 086/173] clickhouse official docker for tests --- .github/workflows/unit-tests.yml | 2 +- .github/workflows/vlm-unit-tests.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index daf1dfe876..78ba5a153b 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -41,7 +41,7 @@ jobs: # Maps tcp port 5432 on service container to the host - 5432:5432 clickhouse: - image: bitnamilegacy/clickhouse:latest + image: clickhouse/clickhouse-server:26.3 ports: - 9000:9000 # Native client interface volumes: diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 6ac26e1a35..bb9ba05a8d 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -25,7 +25,7 @@ jobs: services: clickhouse: - image: bitnamilegacy/clickhouse:latest + image: clickhouse/clickhouse-server:26.3 ports: - 8123:8123 # HTTP interface options: >- From 3d7a68b369cf8275b086457c7a29c119ef410aa6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 11:13:02 -0400 Subject: [PATCH 087/173] test user access managemnt --- .github/workflows/unit-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 78ba5a153b..ccc994c75b 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -57,6 +57,7 @@ jobs: CLICKHOUSE_USER: clickhouse_test CLICKHOUSE_PASSWORD: clickhouse_test ALLOW_EMPTY_PASSWORD: no + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 steps: - uses: actions/checkout@v2 From 67988d81996d6d79b2ff2574cd79737e3a1ddfcb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 11:23:46 -0400 Subject: [PATCH 088/173] add settings --- .github/workflows/unit-tests.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index ccc994c75b..f6ef44807e 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -63,7 +63,7 @@ jobs: - uses: actions/checkout@v2 - name: Set up Clickhouse Settings run: | - cat < test_config.xml + cat < test_user_config.xml @@ -72,10 +72,16 @@ jobs: EOF - docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml + docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml + cat < test_config.xml + + / + + EOF + docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'" - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1 TO clickhouse_test, clickhouse_read_only" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1 TO clickhouse_test, clickhouse_read_only" docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" From 67bc6bf2557eb8d5a93b8559e4083c6805f1490c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 11:25:54 -0400 Subject: [PATCH 089/173] vlm test access maagement --- .github/workflows/vlm-unit-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index bb9ba05a8d..c9e9c22151 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -38,6 +38,7 @@ jobs: CLICKHOUSE_USER: clickhouse_test_user CLICKHOUSE_PASSWORD: clickhouse_test_password ALLOW_EMPTY_PASSWORD: no + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 steps: - uses: actions/checkout@v2 From ddcfa46f466efa3729580322b0c37b8fbd2411d5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 11:27:07 -0400 Subject: [PATCH 090/173] reload config --- .github/workflows/unit-tests.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index f6ef44807e..d609f82caf 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -63,6 +63,13 @@ jobs: - uses: actions/checkout@v2 - name: Set up Clickhouse Settings run: | + cat < test_config.xml + + / + + EOF + docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml + docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD CONFIG" cat < test_user_config.xml @@ -73,12 +80,6 @@ jobs: EOF docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml - cat < test_config.xml - - / - - EOF - docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'" docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1 TO clickhouse_test, clickhouse_read_only" From b8da132de6416489caf08091589accf95140cc9f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 11:32:53 -0400 Subject: [PATCH 091/173] restart clickhouse after confi update --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index d609f82caf..7207ee6514 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -69,7 +69,7 @@ jobs: EOF docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml - docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD CONFIG" + docker restart clickhouse cat < test_user_config.xml From 4f590768d76136c591e77de2485791845168ecda Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 11:41:53 -0400 Subject: [PATCH 092/173] wait for restart --- .github/workflows/unit-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 7207ee6514..9506e06b97 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -70,6 +70,9 @@ jobs: EOF docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml docker restart clickhouse + until docker exec clickhouse clickhouse-client --query "SELECT 1" > /dev/null 2>&1; do + sleep 1 + done cat < test_user_config.xml From 3046d47e42f1f638ce9b577abe0c63aa8dcb7f8c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 11:49:57 -0400 Subject: [PATCH 093/173] explicit sleep for restart --- .github/workflows/unit-tests.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 9506e06b97..f0d7012560 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -70,9 +70,7 @@ jobs: EOF docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml docker restart clickhouse - until docker exec clickhouse clickhouse-client --query "SELECT 1" > /dev/null 2>&1; do - sleep 1 - done + sleep 5 cat < test_user_config.xml From a100ce97d74be105294611b9034978aef05e7e8f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 11:50:51 -0400 Subject: [PATCH 094/173] change order --- .github/workflows/unit-tests.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index f0d7012560..6cd619b785 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -62,15 +62,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Clickhouse Settings - run: | - cat < test_config.xml - - / - - EOF - docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml - docker restart clickhouse - sleep 5 + run: | cat < test_user_config.xml @@ -87,6 +79,13 @@ jobs: docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" + cat < test_config.xml + + / + + EOF + docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml + docker restart clickhouse - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: From f486981de71413b25a24e6a4bde2be9bdd22aa77 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 12:09:16 -0400 Subject: [PATCH 095/173] test with service wait loop --- .github/workflows/unit-tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6cd619b785..c4ac1e2637 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -102,6 +102,14 @@ jobs: python -m pip install --upgrade pip wheel pip install -r requirements.txt pip install -r requirements-dev.txt + - name: Wait for service restart + run: | + timeout 60s sh -c ' + until curl --fail http://localhost:9000; do + echo "Waiting for service to restart..." + sleep 2 + done + ' - name: Run coverage tests run: | export CLICKHOUSE_READER_USER=clickhouse_read_only From 37dfe6bff09fa0140270bbd34db82edf97cc0f59 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 12:11:54 -0400 Subject: [PATCH 096/173] use correct port ofr curl --- .github/workflows/unit-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index c4ac1e2637..e7f35d891d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -44,6 +44,7 @@ jobs: image: clickhouse/clickhouse-server:26.3 ports: - 9000:9000 # Native client interface + - 8123:8123 # HTTP interface volumes: - /var/tmp:/var/seqr/clickhouse-data - /tmp:/in-memory-dir @@ -105,7 +106,7 @@ jobs: - name: Wait for service restart run: | timeout 60s sh -c ' - until curl --fail http://localhost:9000; do + until curl --fail http://localhost:8123; do echo "Waiting for service to restart..." sleep 2 done From cacefff21a433f35a08812705681f9d8a0a6006f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 12:16:31 -0400 Subject: [PATCH 097/173] correct endpoint --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e7f35d891d..934802ece7 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -106,7 +106,7 @@ jobs: - name: Wait for service restart run: | timeout 60s sh -c ' - until curl --fail http://localhost:8123; do + until curl --fail http://localhost:8123/ping; do echo "Waiting for service to restart..." sleep 2 done From 0bde7bf11442d0cd12db838faf3e471848207930 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 12:23:46 -0400 Subject: [PATCH 098/173] seqr hosted clickhouse image --- .github/workflows/unit-tests.yml | 2 +- .github/workflows/vlm-unit-tests.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 934802ece7..5438b01865 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -41,7 +41,7 @@ jobs: # Maps tcp port 5432 on service container to the host - 5432:5432 clickhouse: - image: clickhouse/clickhouse-server:26.3 + image: gcr.io/seqr-project/seqr-clickhouse:26.3.9 ports: - 9000:9000 # Native client interface - 8123:8123 # HTTP interface diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index c9e9c22151..f5b1942b2e 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -25,7 +25,7 @@ jobs: services: clickhouse: - image: clickhouse/clickhouse-server:26.3 + image: gcr.io/seqr-project/seqr-clickhouse:26.3.9 ports: - 8123:8123 # HTTP interface options: >- From 695408f99d66b0b135ee6834c135134e8201c293 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 12:28:59 -0400 Subject: [PATCH 099/173] health check clean up --- .github/workflows/unit-tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 5438b01865..55b0cfaa73 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -105,10 +105,9 @@ jobs: pip install -r requirements-dev.txt - name: Wait for service restart run: | - timeout 60s sh -c ' + timeout 50s sh -c ' until curl --fail http://localhost:8123/ping; do - echo "Waiting for service to restart..." - sleep 2 + sleep 5 done ' - name: Run coverage tests From 686e30070a13eac095248ac6711325d14eb6df70 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 14:15:40 -0400 Subject: [PATCH 100/173] debug test failure --- .github/workflows/unit-tests.yml | 2 +- seqr/management/tests/tag_seqr_prioritized_variants_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 55b0cfaa73..ebfbaeb622 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -115,7 +115,7 @@ jobs: export CLICKHOUSE_READER_USER=clickhouse_read_only export CLICKHOUSE_WRITER_USER=clickhouse_test export CLICKHOUSE_SERVICE_HOSTNAME=localhost - coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data clickhouse_search seqr matchmaker panelapp + coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 seqr.management.tests.tag_seqr_prioritized_variants_tests coverage report -m --fail-under=99 coverage report | (! grep [1-8][0-9]%) coverage report | (! grep 9[0-5]%) diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index 90c13c8e65..41f088af30 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -133,13 +133,13 @@ def test_command(self, mock_datetime, mock_slack, mock_email): mock_email.reset_mock() mock_slack.reset_mock() call_command('tag_seqr_prioritized_variants', PROJECT_GUID) - self._assert_expected_logs(num_unchanged=7) mock_email.assert_not_called() mock_slack.assert_not_called() self.assertDictEqual(expected_tags, { tuple(sorted(tag.saved_variants.values_list('key', flat=True))): json.loads(tag.metadata) for tag in VariantTag.objects.filter(variant_tag_type__name='seqr Prioritized') }) + self._assert_expected_logs(num_unchanged=7) def _assert_expected_logs(self, num_new=0, num_unchanged=0, creation_stats=None): creation_stats = creation_stats or {} From 0e2a4d427f4eadb8b7e739455b4e059d94708928 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 14:34:54 -0400 Subject: [PATCH 101/173] debug test failure --- seqr/management/tests/tag_seqr_prioritized_variants_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index 41f088af30..68c57c5efe 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -133,13 +133,15 @@ def test_command(self, mock_datetime, mock_slack, mock_email): mock_email.reset_mock() mock_slack.reset_mock() call_command('tag_seqr_prioritized_variants', PROJECT_GUID) + self.maxDiff = None + self.assertListEqual([json.loads(log) for log in self._log_stream.getvalue().split('\n') if log], []) + self._assert_expected_logs(num_unchanged=7) mock_email.assert_not_called() mock_slack.assert_not_called() self.assertDictEqual(expected_tags, { tuple(sorted(tag.saved_variants.values_list('key', flat=True))): json.loads(tag.metadata) for tag in VariantTag.objects.filter(variant_tag_type__name='seqr Prioritized') }) - self._assert_expected_logs(num_unchanged=7) def _assert_expected_logs(self, num_new=0, num_unchanged=0, creation_stats=None): creation_stats = creation_stats or {} From 41267eef6dec8717dcc52a74ddf5a3b3ca554cc7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 14:47:03 -0400 Subject: [PATCH 102/173] debug test failure --- .../tag_seqr_prioritized_variants_tests.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index 68c57c5efe..25549859d3 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -134,7 +134,27 @@ def test_command(self, mock_datetime, mock_slack, mock_email): mock_slack.reset_mock() call_command('tag_seqr_prioritized_variants', PROJECT_GUID) self.maxDiff = None - self.assertListEqual([json.loads(log) for log in self._log_stream.getvalue().split('\n') if log], []) + expected = [{ + 'timestamp': mock.ANY, 'severity': 'INFO', 'message': log, **(extra or {}), + } for log, extra in self._dataset_type_logs( + 'SNV_INDEL', 3, SNV_INDEL_MATCHES, **creation_stats.get('SNV_INDEL', {}), + ) + self._dataset_type_logs( + 'SV_WES', 1, SV_MATCHES, **creation_stats.get('SV', {}), + ) + self._dataset_type_logs( + 'MITO', 1, MITO_MATCHES, **creation_stats.get('MITO', {}), + ) + self._dataset_type_logs( + 'multi data type', 1, MULTI_TYPE_MATCHES, **creation_stats.get('MULTI', {}), + search_dataset_types=['SNV_INDEL', 'SNV_INDEL/SV_WES'], + ) + [ + (f'Tagged {num_new} new and 0 previously tagged variants in 1 families, found {num_unchanged} unchanged tags:', + None) + ] + [(f' {criteria}: {count} variants', None) for criteria, (count, _) in + SNV_INDEL_MATCHES.items()] + [ + (f' {criteria}: {count} variants', None) for criteria, (count, _) in SV_MATCHES.items() + ] + [(f' {criteria}: {count} variants', None) for criteria, (count, _) in MITO_MATCHES.items()] + [ + (f' {criteria}: {count} variants', None) for criteria, (count, _) in MULTI_TYPE_MATCHES.items() + ]] + self.assertListEqual([json.loads(log) for log in self._log_stream.getvalue().split('\n') if log], expected) self._assert_expected_logs(num_unchanged=7) mock_email.assert_not_called() mock_slack.assert_not_called() From b7e5fad75a10c07ec6a82a90ebcb09a8aa091d6d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 14:54:45 -0400 Subject: [PATCH 103/173] debug test failure --- .../tests/tag_seqr_prioritized_variants_tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index 25549859d3..9ce9860864 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -137,16 +137,16 @@ def test_command(self, mock_datetime, mock_slack, mock_email): expected = [{ 'timestamp': mock.ANY, 'severity': 'INFO', 'message': log, **(extra or {}), } for log, extra in self._dataset_type_logs( - 'SNV_INDEL', 3, SNV_INDEL_MATCHES, **creation_stats.get('SNV_INDEL', {}), + 'SNV_INDEL', 3, SNV_INDEL_MATCHES, ) + self._dataset_type_logs( - 'SV_WES', 1, SV_MATCHES, **creation_stats.get('SV', {}), + 'SV_WES', 1, SV_MATCHES, ) + self._dataset_type_logs( - 'MITO', 1, MITO_MATCHES, **creation_stats.get('MITO', {}), + 'MITO', 1, MITO_MATCHES, ) + self._dataset_type_logs( - 'multi data type', 1, MULTI_TYPE_MATCHES, **creation_stats.get('MULTI', {}), + 'multi data type', 1, MULTI_TYPE_MATCHES, search_dataset_types=['SNV_INDEL', 'SNV_INDEL/SV_WES'], ) + [ - (f'Tagged {num_new} new and 0 previously tagged variants in 1 families, found {num_unchanged} unchanged tags:', + (f'Tagged 0 new and 0 previously tagged variants in 1 families, found 7 unchanged tags:', None) ] + [(f' {criteria}: {count} variants', None) for criteria, (count, _) in SNV_INDEL_MATCHES.items()] + [ From d9c4fcda43dbe23647409e8bd2ec8ccff658c10a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 17:18:12 -0400 Subject: [PATCH 104/173] try cache settings --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index ebfbaeb622..ae92e5f898 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -76,7 +76,7 @@ jobs: docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'" - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1 TO clickhouse_test, clickhouse_read_only" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, use_query_condition_cache=0, use_statistics_cache=0 TO clickhouse_test, clickhouse_read_only" docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" From becdaaa1e90a30e62b4e1650ad1e7c9f39c78f00 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 17:23:43 -0400 Subject: [PATCH 105/173] settings iter --- .github/workflows/unit-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index ae92e5f898..1d16cb5b37 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -76,7 +76,8 @@ jobs: docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'" - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, use_query_condition_cache=0, use_statistics_cache=0 TO clickhouse_test, clickhouse_read_only" +# use_query_condition_cache, use_statistics_cache + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, use_join_disjunctions_push_down=0 TO clickhouse_test, clickhouse_read_only" docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" From 5c306ca6b5458b80e986a15f975c03c95c943693 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 17:25:40 -0400 Subject: [PATCH 106/173] settings iter --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 1d16cb5b37..0280c3d809 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -77,7 +77,7 @@ jobs: docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'" # use_query_condition_cache, use_statistics_cache - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, use_join_disjunctions_push_down=0 TO clickhouse_test, clickhouse_read_only" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, use_join_disjunctions_push_down=0, allow_experimental_analyzer=0 TO clickhouse_test, clickhouse_read_only" docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" From ce4a4bab3b90c0c7751f396fdb1a3b231e527e61 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 27 May 2026 17:30:24 -0400 Subject: [PATCH 107/173] settings iter --- .github/workflows/unit-tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 0280c3d809..1b36809aeb 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -76,8 +76,7 @@ jobs: docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'" -# use_query_condition_cache, use_statistics_cache - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, use_join_disjunctions_push_down=0, allow_experimental_analyzer=0 TO clickhouse_test, clickhouse_read_only" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, use_join_disjunctions_push_down=0, allow_experimental_analyzer=0, allow_general_join_planning=0, allow_reorder_prewhere_conditions=0 TO clickhouse_test, clickhouse_read_only" docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" From b931447226ccc8cfb7675cca6e5b6b50ed956c57 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 11:24:44 -0400 Subject: [PATCH 108/173] settings iter --- .github/workflows/unit-tests.yml | 2 +- seqr/management/tests/tag_seqr_prioritized_variants_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 1b36809aeb..a6c4c9348b 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -76,7 +76,7 @@ jobs: docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'" - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, use_join_disjunctions_push_down=0, allow_experimental_analyzer=0, allow_general_join_planning=0, allow_reorder_prewhere_conditions=0 TO clickhouse_test, clickhouse_read_only" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1 TO clickhouse_test, clickhouse_read_only" docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index 9ce9860864..7f2fa12bcc 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -8,7 +8,7 @@ from seqr.models import SavedVariant, VariantTag PROJECT_GUID = 'R0001_1kg' - +# use_join_disjunctions_push_down=0, allow_experimental_analyzer=0, allow_general_join_planning=0, allow_reorder_prewhere_conditions=0 SNV_INDEL_MATCHES = { 'Clinvar Pathogenic': (0, None), 'Clinvar Pathogenic - Compound Heterozygous': (0, None), From 22ad270e71f4564293d1a0463277754bed7c36f5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 11:29:11 -0400 Subject: [PATCH 109/173] settings iter --- .github/workflows/unit-tests.yml | 2 +- seqr/management/tests/tag_seqr_prioritized_variants_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index a6c4c9348b..55cc9e7415 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -76,7 +76,7 @@ jobs: docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'" - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1 TO clickhouse_test, clickhouse_read_only" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, use_join_disjunctions_push_down=0 TO clickhouse_test, clickhouse_read_only" docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index 7f2fa12bcc..2eb5d2e3a6 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -8,7 +8,7 @@ from seqr.models import SavedVariant, VariantTag PROJECT_GUID = 'R0001_1kg' -# use_join_disjunctions_push_down=0, allow_experimental_analyzer=0, allow_general_join_planning=0, allow_reorder_prewhere_conditions=0 +# allow_experimental_analyzer=0, allow_general_join_planning=0, allow_reorder_prewhere_conditions=0 SNV_INDEL_MATCHES = { 'Clinvar Pathogenic': (0, None), 'Clinvar Pathogenic - Compound Heterozygous': (0, None), From 737eeb3498e608b19d48977769efcc43d18f25f0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 11:33:17 -0400 Subject: [PATCH 110/173] settings iter --- .github/workflows/unit-tests.yml | 2 +- seqr/management/tests/tag_seqr_prioritized_variants_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 55cc9e7415..c0c90ead55 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -76,7 +76,7 @@ jobs: docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'" - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, use_join_disjunctions_push_down=0 TO clickhouse_test, clickhouse_read_only" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, allow_reorder_prewhere_conditions=0 TO clickhouse_test, clickhouse_read_only" docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index 2eb5d2e3a6..99a3cad624 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -8,7 +8,7 @@ from seqr.models import SavedVariant, VariantTag PROJECT_GUID = 'R0001_1kg' -# allow_experimental_analyzer=0, allow_general_join_planning=0, allow_reorder_prewhere_conditions=0 +# allow_experimental_analyzer=0, allow_general_join_planning=0, SNV_INDEL_MATCHES = { 'Clinvar Pathogenic': (0, None), 'Clinvar Pathogenic - Compound Heterozygous': (0, None), From 96ff9df1be128e4e6e4d3ec82deaa6a89de093b5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 11:38:28 -0400 Subject: [PATCH 111/173] settings iter --- .github/workflows/unit-tests.yml | 2 +- seqr/management/tests/tag_seqr_prioritized_variants_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index c0c90ead55..46f6699724 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -76,7 +76,7 @@ jobs: docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'" - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, allow_reorder_prewhere_conditions=0 TO clickhouse_test, clickhouse_read_only" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, allow_experimental_analyzer=0 TO clickhouse_test, clickhouse_read_only" docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index 99a3cad624..185524417e 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -8,7 +8,7 @@ from seqr.models import SavedVariant, VariantTag PROJECT_GUID = 'R0001_1kg' -# allow_experimental_analyzer=0, allow_general_join_planning=0, +# allow_general_join_planning=0, enable_join_runtime_filters=0, enable_lazy_columns_replication=0 SNV_INDEL_MATCHES = { 'Clinvar Pathogenic': (0, None), 'Clinvar Pathogenic - Compound Heterozygous': (0, None), From 4ac1ef8d80bb544c26cb3ecdc5f9b804c5fb9128 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 11:43:44 -0400 Subject: [PATCH 112/173] settings iter --- .github/workflows/unit-tests.yml | 2 +- seqr/management/tests/tag_seqr_prioritized_variants_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 46f6699724..bd8e630fd7 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -76,7 +76,7 @@ jobs: docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'" - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, allow_experimental_analyzer=0 TO clickhouse_test, clickhouse_read_only" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, allow_general_join_planning=0, enable_join_runtime_filters=0, enable_lazy_columns_replication=0 TO clickhouse_test, clickhouse_read_only" docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index 185524417e..9ce9860864 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -8,7 +8,7 @@ from seqr.models import SavedVariant, VariantTag PROJECT_GUID = 'R0001_1kg' -# allow_general_join_planning=0, enable_join_runtime_filters=0, enable_lazy_columns_replication=0 + SNV_INDEL_MATCHES = { 'Clinvar Pathogenic': (0, None), 'Clinvar Pathogenic - Compound Heterozygous': (0, None), From 0cca691145a488f82b0c8c46c3d2331b6d5b159a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 11:48:32 -0400 Subject: [PATCH 113/173] settings iter --- .github/workflows/unit-tests.yml | 2 +- seqr/management/tests/tag_seqr_prioritized_variants_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index bd8e630fd7..ab6a575454 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -76,7 +76,7 @@ jobs: docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'" - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, allow_general_join_planning=0, enable_join_runtime_filters=0, enable_lazy_columns_replication=0 TO clickhouse_test, clickhouse_read_only" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, allow_general_join_planning=0 TO clickhouse_test, clickhouse_read_only" docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index 9ce9860864..5500671590 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -8,7 +8,7 @@ from seqr.models import SavedVariant, VariantTag PROJECT_GUID = 'R0001_1kg' - +# enable_join_runtime_filters=0, enable_lazy_columns_replication=0 SNV_INDEL_MATCHES = { 'Clinvar Pathogenic': (0, None), 'Clinvar Pathogenic - Compound Heterozygous': (0, None), From 799c93f9d8392ba05234fc1800681cea97a7be8a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 11:53:06 -0400 Subject: [PATCH 114/173] settings iter --- .github/workflows/unit-tests.yml | 2 +- seqr/management/tests/tag_seqr_prioritized_variants_tests.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index ab6a575454..5afe4aa358 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -76,7 +76,7 @@ jobs: docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'" - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, allow_general_join_planning=0 TO clickhouse_test, clickhouse_read_only" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, enable_join_runtime_filters=0 TO clickhouse_test, clickhouse_read_only" docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index 5500671590..b5156ddf37 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -8,7 +8,8 @@ from seqr.models import SavedVariant, VariantTag PROJECT_GUID = 'R0001_1kg' -# enable_join_runtime_filters=0, enable_lazy_columns_replication=0 +# allow_general_join_planning=0 +# , enable_lazy_columns_replication=0 SNV_INDEL_MATCHES = { 'Clinvar Pathogenic': (0, None), 'Clinvar Pathogenic - Compound Heterozygous': (0, None), From 8a18d9302afa71f4f16ce0e8d84e17432101975b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 11:58:52 -0400 Subject: [PATCH 115/173] clean up --- .github/workflows/unit-tests.yml | 2 +- .../tag_seqr_prioritized_variants_tests.py | 25 +------------------ 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 5afe4aa358..fae9c684e8 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -115,7 +115,7 @@ jobs: export CLICKHOUSE_READER_USER=clickhouse_read_only export CLICKHOUSE_WRITER_USER=clickhouse_test export CLICKHOUSE_SERVICE_HOSTNAME=localhost - coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 seqr.management.tests.tag_seqr_prioritized_variants_tests + coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data clickhouse_search seqr matchmaker panelapp coverage report -m --fail-under=99 coverage report | (! grep [1-8][0-9]%) coverage report | (! grep 9[0-5]%) diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index b5156ddf37..90c13c8e65 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -8,8 +8,7 @@ from seqr.models import SavedVariant, VariantTag PROJECT_GUID = 'R0001_1kg' -# allow_general_join_planning=0 -# , enable_lazy_columns_replication=0 + SNV_INDEL_MATCHES = { 'Clinvar Pathogenic': (0, None), 'Clinvar Pathogenic - Compound Heterozygous': (0, None), @@ -134,28 +133,6 @@ def test_command(self, mock_datetime, mock_slack, mock_email): mock_email.reset_mock() mock_slack.reset_mock() call_command('tag_seqr_prioritized_variants', PROJECT_GUID) - self.maxDiff = None - expected = [{ - 'timestamp': mock.ANY, 'severity': 'INFO', 'message': log, **(extra or {}), - } for log, extra in self._dataset_type_logs( - 'SNV_INDEL', 3, SNV_INDEL_MATCHES, - ) + self._dataset_type_logs( - 'SV_WES', 1, SV_MATCHES, - ) + self._dataset_type_logs( - 'MITO', 1, MITO_MATCHES, - ) + self._dataset_type_logs( - 'multi data type', 1, MULTI_TYPE_MATCHES, - search_dataset_types=['SNV_INDEL', 'SNV_INDEL/SV_WES'], - ) + [ - (f'Tagged 0 new and 0 previously tagged variants in 1 families, found 7 unchanged tags:', - None) - ] + [(f' {criteria}: {count} variants', None) for criteria, (count, _) in - SNV_INDEL_MATCHES.items()] + [ - (f' {criteria}: {count} variants', None) for criteria, (count, _) in SV_MATCHES.items() - ] + [(f' {criteria}: {count} variants', None) for criteria, (count, _) in MITO_MATCHES.items()] + [ - (f' {criteria}: {count} variants', None) for criteria, (count, _) in MULTI_TYPE_MATCHES.items() - ]] - self.assertListEqual([json.loads(log) for log in self._log_stream.getvalue().split('\n') if log], expected) self._assert_expected_logs(num_unchanged=7) mock_email.assert_not_called() mock_slack.assert_not_called() From 90fefd0c977b47d671a311581b59a6aecd62a6c0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 13:54:36 -0400 Subject: [PATCH 116/173] setup for bumped version --- .github/workflows/vlm-unit-tests.yaml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 2f345ec859..e1c10a750b 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -69,7 +69,7 @@ jobs: pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Settings run: | - cat < test_config.xml + cat < test_user_config.xml @@ -81,11 +81,25 @@ jobs: EOF - docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml + docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS allow_materialized_view_with_bad_select=1, stop_refreshable_materialized_views_on_startup=1 TO vlm_test_user, clickhouse_test_user" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" + cat < test_config.xml + + / + + EOF + docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml + docker restart clickhouse + - name: Wait for service restart + run: | + timeout 50s sh -c ' + until curl --fail http://localhost:8123/ping; do + sleep 5 + done + ' - name: Set up Clickhouse Test Database run: | python3 -m venv .venv From d2f451c18da1c927b0cea0a33884ef5287e170f2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 14:09:40 -0400 Subject: [PATCH 117/173] adjust settings profile --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index e1c10a750b..f6436fc172 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -83,7 +83,7 @@ jobs: EOF docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS allow_materialized_view_with_bad_select=1, stop_refreshable_materialized_views_on_startup=1 TO vlm_test_user, clickhouse_test_user" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, enable_join_runtime_filters=0 TO vlm_test_user, clickhouse_test_user" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" cat < test_config.xml From 915f7ccf7bdef12f49505b1f5476ffaaaa53846e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 14:18:10 -0400 Subject: [PATCH 118/173] run without pytest setup --- .github/workflows/vlm-unit-tests.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index f6436fc172..69b5b8faf8 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -113,7 +113,9 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -m pytest --setup-only --create-db --reuse-db --ds=settings -vv + python3 -u manage.py migrate + python3 -u manage.py migrate --database=reference_data + python3 -u manage.py migrate --database=clickhouse_write python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse test user run: | From 66e0f6b6d8bbdc7d37b42fbc3d0a977293e7a671 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 14:21:51 -0400 Subject: [PATCH 119/173] add dbs --- .github/workflows/vlm-unit-tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 69b5b8faf8..cf7355f01d 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -42,6 +42,7 @@ jobs: env: CLICKHOUSE_USER: clickhouse_test_user CLICKHOUSE_PASSWORD: clickhouse_test_password + CLICKHOUSE_DB: seqr ALLOW_EMPTY_PASSWORD: no CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 postgres: @@ -55,6 +56,7 @@ jobs: --health-retries 5 env: POSTGRES_PASSWORD: pgtest + POSTGRES_DB: seqrdb steps: - name: Set up Python From 964a42e584a17e6d8c0f663fe95cfac10a18bc59 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 14:30:38 -0400 Subject: [PATCH 120/173] initialize reference data db --- .github/workflows/vlm-unit-tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index cf7355f01d..2405fce9ce 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -102,7 +102,7 @@ jobs: sleep 5 done ' - - name: Set up Clickhouse Test Database + - name: Set up Test Databases run: | python3 -m venv .venv source .venv/bin/activate @@ -115,6 +115,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost + docker exec -it postgres createdb reference_data_db python3 -u manage.py migrate python3 -u manage.py migrate --database=reference_data python3 -u manage.py migrate --database=clickhouse_write From 6b4e630ffb6e77cdc8c5665c134f163298007e16 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 14:38:48 -0400 Subject: [PATCH 121/173] skip postgres setup --- .github/workflows/vlm-unit-tests.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 2405fce9ce..a83f502bc3 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -56,7 +56,6 @@ jobs: --health-retries 5 env: POSTGRES_PASSWORD: pgtest - POSTGRES_DB: seqrdb steps: - name: Set up Python @@ -115,9 +114,6 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - docker exec -it postgres createdb reference_data_db - python3 -u manage.py migrate - python3 -u manage.py migrate --database=reference_data python3 -u manage.py migrate --database=clickhouse_write python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse test user From 5810451257fe58e0ac6346a48503fed5b02fe8b3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 14:42:54 -0400 Subject: [PATCH 122/173] reference data db --- .github/workflows/vlm-unit-tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index a83f502bc3..063231b5e2 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -56,6 +56,7 @@ jobs: --health-retries 5 env: POSTGRES_PASSWORD: pgtest + POSTGRES_DB: seqrdb steps: - name: Set up Python @@ -103,6 +104,8 @@ jobs: ' - name: Set up Test Databases run: | + docker container ls + docker exec postgres createdb reference_data_db python3 -m venv .venv source .venv/bin/activate pip install -r vlm/requirements.txt From 896fb22aa899beea5bb6402d121e8db10a4fc5ad Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 14:49:05 -0400 Subject: [PATCH 123/173] postgres iage name --- .github/workflows/vlm-unit-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 063231b5e2..93e8252f60 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -54,6 +54,7 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + --name postgres env: POSTGRES_PASSWORD: pgtest POSTGRES_DB: seqrdb From 6816ef40cb793e3a366b969c85e6020e2a9e388f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 14:50:46 -0400 Subject: [PATCH 124/173] fix reference data db reate --- .github/workflows/vlm-unit-tests.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 93e8252f60..29f9790830 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -105,8 +105,7 @@ jobs: ' - name: Set up Test Databases run: | - docker container ls - docker exec postgres createdb reference_data_db + docker exec postgres createdb -U postgres reference_data_db python3 -m venv .venv source .venv/bin/activate pip install -r vlm/requirements.txt From 06701acd7b721793f2ee056e5206bcb973e0f9b4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 15:00:09 -0400 Subject: [PATCH 125/173] create clickhouse db --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 29f9790830..949839e1fb 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -42,7 +42,6 @@ jobs: env: CLICKHOUSE_USER: clickhouse_test_user CLICKHOUSE_PASSWORD: clickhouse_test_password - CLICKHOUSE_DB: seqr ALLOW_EMPTY_PASSWORD: no CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 postgres: @@ -106,6 +105,7 @@ jobs: - name: Set up Test Databases run: | docker exec postgres createdb -U postgres reference_data_db + docker exec clickhouse clickhouse-client --query "CREATE DATABASE seqr" python3 -m venv .venv source .venv/bin/activate pip install -r vlm/requirements.txt From 273469945af3d194a09343f449d4afb983dda870 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 28 May 2026 15:24:13 -0400 Subject: [PATCH 126/173] back to pytest --- .github/workflows/vlm-unit-tests.yaml | 6 +----- vlm/conftest.py | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 949839e1fb..cf79d885aa 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -53,10 +53,8 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - --name postgres env: POSTGRES_PASSWORD: pgtest - POSTGRES_DB: seqrdb steps: - name: Set up Python @@ -104,8 +102,6 @@ jobs: ' - name: Set up Test Databases run: | - docker exec postgres createdb -U postgres reference_data_db - docker exec clickhouse clickhouse-client --query "CREATE DATABASE seqr" python3 -m venv .venv source .venv/bin/activate pip install -r vlm/requirements.txt @@ -117,7 +113,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -u manage.py migrate --database=clickhouse_write + python3 -m pytest --setup-only --create-db --reuse-db --ds=settings -vv python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse test user run: | diff --git a/vlm/conftest.py b/vlm/conftest.py index e6d7a5220a..da4fe62416 100644 --- a/vlm/conftest.py +++ b/vlm/conftest.py @@ -12,7 +12,7 @@ def django_db_setup(request, django_db_blocker,django_db_keepdb): db_cfg = setup_databases( verbosity=request.config.option.verbose, interactive=False, - aliases=['default', 'clickhouse_write'], + aliases=['default', 'reference_data', 'clickhouse_write'], keepdb=django_db_keepdb, ) # TODO loaddata From c851d3a5aaaebc6935d380ec9d01157ec752867d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 10:42:00 -0400 Subject: [PATCH 127/173] create dbs --- .github/workflows/vlm-unit-tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index cf79d885aa..d8453f18f5 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -102,6 +102,8 @@ jobs: ' - name: Set up Test Databases run: | + docker exec postgres createdb -U postgres reference_data_db + docker exec clickhouse clickhouse-client --query "CREATE DATABASE seqr" python3 -m venv .venv source .venv/bin/activate pip install -r vlm/requirements.txt From b3394fd760b039d6c376aaa277603e284d9b898c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 10:44:28 -0400 Subject: [PATCH 128/173] fix loaddata --- .github/workflows/vlm-unit-tests.yaml | 3 --- vlm/conftest.py | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index d8453f18f5..6827b141a4 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -102,8 +102,6 @@ jobs: ' - name: Set up Test Databases run: | - docker exec postgres createdb -U postgres reference_data_db - docker exec clickhouse clickhouse-client --query "CREATE DATABASE seqr" python3 -m venv .venv source .venv/bin/activate pip install -r vlm/requirements.txt @@ -116,7 +114,6 @@ jobs: export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost python3 -m pytest --setup-only --create-db --reuse-db --ds=settings -vv - python3 -u manage.py loaddata clickhouse_search --database=clickhouse_write - name: Set up Clickhouse test user run: | docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" diff --git a/vlm/conftest.py b/vlm/conftest.py index da4fe62416..1c610862ba 100644 --- a/vlm/conftest.py +++ b/vlm/conftest.py @@ -6,6 +6,7 @@ def enable_db_access_for_all_tests(db): @pytest.fixture(scope='session') def django_db_setup(request, django_db_blocker,django_db_keepdb): + from django.core.management import call_command from django.test.utils import setup_databases, teardown_databases with django_db_blocker.unblock(): @@ -15,7 +16,7 @@ def django_db_setup(request, django_db_blocker,django_db_keepdb): aliases=['default', 'reference_data', 'clickhouse_write'], keepdb=django_db_keepdb, ) - # TODO loaddata + call_command('loaddata', 'clickhouse_search', '--database=clickhouse_write') yield From 959cf85af071853a2728c1d5b69130257186338c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 10:51:12 -0400 Subject: [PATCH 129/173] debug clickhouse users --- .github/workflows/vlm-unit-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 6827b141a4..fee5635b03 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -86,6 +86,7 @@ jobs: docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, enable_join_runtime_filters=0 TO vlm_test_user, clickhouse_test_user" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" + docker exec clickhouse clickhouse-client --query "SHOW USERS" cat < test_config.xml / From 7f309d28ca851d452e0d9f79f00a12218b262226 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 10:55:34 -0400 Subject: [PATCH 130/173] remove debug --- .github/workflows/vlm-unit-tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index fee5635b03..6827b141a4 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -86,7 +86,6 @@ jobs: docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, enable_join_runtime_filters=0 TO vlm_test_user, clickhouse_test_user" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - docker exec clickhouse clickhouse-client --query "SHOW USERS" cat < test_config.xml / From 5fb72973e2bca7e55a9ed928926d69d415996565 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 10:58:38 -0400 Subject: [PATCH 131/173] escape chars --- .github/workflows/vlm-unit-tests.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 6827b141a4..2a5db63bf6 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -116,10 +116,10 @@ jobs: python3 -m pytest --setup-only --create-db --reuse-db --ds=settings -vv - name: Set up Clickhouse test user run: | - docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user" - docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.\`GRCh37/SNV_INDEL/key_lookup\` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.\`GRCh38/SNV_INDEL/key_lookup\` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.\`GRCh37/SNV_INDEL/gt_stats_dict\` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.\`GRCh38/SNV_INDEL/gt_stats_dict\` TO vlm_test_user" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - name: Run coverage tests run: | From d0b20cf3443bbaced2495118441da01d18a8fa5c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 11:04:49 -0400 Subject: [PATCH 132/173] fix user creation --- .github/workflows/vlm-unit-tests.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 2a5db63bf6..7568055894 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -75,13 +75,11 @@ jobs: 1 - - vlm_test_password - EOF docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml + docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, enable_join_runtime_filters=0 TO vlm_test_user, clickhouse_test_user" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" From f6480a7de43d375f9d079b1cfbb487b978619ac3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 11:10:38 -0400 Subject: [PATCH 133/173] remove invalid arg --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 7568055894..86dd4089e9 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -128,6 +128,6 @@ jobs: export CLICKHOUSE_VLM_USERNAME=vlm_test_user export CLICKHOUSE_VLM_PASSWORD=vlm_test_password export CLICKHOUSE_DATABASE=test_seqr - coverage run --source="./vlm" --omit="./vlm/__main__.py" -m pytest vlm/ --reuse-db + coverage run --source="./vlm" --omit="./vlm/__main__.py" -m pytest vlm/ coverage report -m --fail-under=95 From 7e441c081956f449c5ca9d773d2fb51e347bab2e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 11:11:04 -0400 Subject: [PATCH 134/173] lower verbosity --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 86dd4089e9..c26c7ab070 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -111,7 +111,7 @@ jobs: export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password export CLICKHOUSE_SERVICE_HOSTNAME=localhost export POSTGRES_SERVICE_HOSTNAME=localhost - python3 -m pytest --setup-only --create-db --reuse-db --ds=settings -vv + python3 -m pytest --setup-only --create-db --reuse-db --ds=settings - name: Set up Clickhouse test user run: | docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.\`GRCh37/SNV_INDEL/key_lookup\` TO vlm_test_user" From ff56d59c5dd958609997d640599ea55b6dc9df51 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 11:20:09 -0400 Subject: [PATCH 135/173] Revert "setup for bumped version" This reverts commit 90fefd0c977b47d671a311581b59a6aecd62a6c0. --- .github/workflows/vlm-unit-tests.yaml | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index c26c7ab070..c628d71956 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -69,7 +69,7 @@ jobs: pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Settings run: | - cat < test_user_config.xml + cat < test_config.xml @@ -78,27 +78,13 @@ jobs: EOF - docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml + docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, enable_join_runtime_filters=0 TO vlm_test_user, clickhouse_test_user" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - cat < test_config.xml - - / - - EOF - docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml - docker restart clickhouse - - name: Wait for service restart - run: | - timeout 50s sh -c ' - until curl --fail http://localhost:8123/ping; do - sleep 5 - done - ' - - name: Set up Test Databases + - name: Set up Clickhouse Test Database run: | python3 -m venv .venv source .venv/bin/activate From 49d06f469aeac9337bbb3784a8fd2bad176e3853 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 11:20:41 -0400 Subject: [PATCH 136/173] Revert "Merge branch 'clickhouse-bump-unit-test-version' of https://github.com/broadinstitute/seqr into vlm-shared-test-db" This reverts commit 0491d230d2d4ca35ab60ca3dec6cf2aa1f582538, reversing changes made to cfcee0423089efed5ced695c9a62a931f8a8c939. --- .github/workflows/unit-tests.yml | 26 +-- .github/workflows/vlm-unit-tests.yaml | 3 +- ui/package-lock.json | 269 ++++++++------------------ vlm/match.py | 28 +-- vlm/requirements-test.txt | 6 +- 5 files changed, 96 insertions(+), 236 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index fae9c684e8..daf1dfe876 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -41,10 +41,9 @@ jobs: # Maps tcp port 5432 on service container to the host - 5432:5432 clickhouse: - image: gcr.io/seqr-project/seqr-clickhouse:26.3.9 + image: bitnamilegacy/clickhouse:latest ports: - 9000:9000 # Native client interface - - 8123:8123 # HTTP interface volumes: - /var/tmp:/var/seqr/clickhouse-data - /tmp:/in-memory-dir @@ -58,13 +57,12 @@ jobs: CLICKHOUSE_USER: clickhouse_test CLICKHOUSE_PASSWORD: clickhouse_test ALLOW_EMPTY_PASSWORD: no - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 steps: - uses: actions/checkout@v2 - name: Set up Clickhouse Settings - run: | - cat < test_user_config.xml + run: | + cat < test_config.xml @@ -73,20 +71,13 @@ jobs: EOF - docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml + docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'" - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, enable_join_runtime_filters=0 TO clickhouse_test, clickhouse_read_only" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1 TO clickhouse_test, clickhouse_read_only" docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - cat < test_config.xml - - / - - EOF - docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml - docker restart clickhouse - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: @@ -103,13 +94,6 @@ jobs: python -m pip install --upgrade pip wheel pip install -r requirements.txt pip install -r requirements-dev.txt - - name: Wait for service restart - run: | - timeout 50s sh -c ' - until curl --fail http://localhost:8123/ping; do - sleep 5 - done - ' - name: Run coverage tests run: | export CLICKHOUSE_READER_USER=clickhouse_read_only diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index c628d71956..3c10b95629 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -26,7 +26,7 @@ jobs: services: clickhouse: - image: gcr.io/seqr-project/seqr-clickhouse:26.3.9 + image: bitnamilegacy/clickhouse:latest ports: - 8123:8123 # HTTP interface - 9000:9000 # Native client interface @@ -43,7 +43,6 @@ jobs: CLICKHOUSE_USER: clickhouse_test_user CLICKHOUSE_PASSWORD: clickhouse_test_password ALLOW_EMPTY_PASSWORD: no - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 postgres: image: postgres ports: diff --git a/ui/package-lock.json b/ui/package-lock.json index 21e4d98631..f58320ac31 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -4707,23 +4707,23 @@ } }, "node_modules/body-parser": { - "version": "1.20.5", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", - "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", + "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", "dev": true, "dependencies": { - "bytes": "~3.1.2", + "bytes": "3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", - "destroy": "~1.2.0", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "on-finished": "~2.4.1", - "qs": "~6.15.1", - "raw-body": "~2.5.3", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.13.0", + "raw-body": "2.5.2", "type-is": "~1.6.18", - "unpipe": "~1.0.0" + "unpipe": "1.0.0" }, "engines": { "node": ">= 0.8", @@ -4739,41 +4739,6 @@ "node": ">= 0.8" } }, - "node_modules/body-parser/node_modules/http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "dev": true, - "dependencies": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - }, - "engines": { - "node": ">= 0.8" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/body-parser/node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true - }, - "node_modules/body-parser/node_modules/statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/bonjour-service": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/bonjour-service/-/bonjour-service-1.3.0.tgz", @@ -8031,14 +7996,14 @@ } }, "node_modules/express": { - "version": "4.22.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", - "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", "dev": true, "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "~1.20.5", + "body-parser": "~1.20.3", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", @@ -8057,7 +8022,7 @@ "parseurl": "~1.3.3", "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", - "qs": "~6.15.1", + "qs": "~6.14.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", @@ -8106,6 +8071,21 @@ "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "dev": true }, + "node_modules/express/node_modules/qs": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "dev": true, + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/express/node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -14563,12 +14543,12 @@ } }, "node_modules/qs": { - "version": "6.15.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", - "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", "dev": true, "dependencies": { - "side-channel": "^1.1.0" + "side-channel": "^1.0.6" }, "engines": { "node": ">=0.6" @@ -14665,60 +14645,16 @@ } }, "node_modules/raw-body": { - "version": "2.5.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", - "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", - "dev": true, - "dependencies": { - "bytes": "~3.1.2", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/raw-body/node_modules/depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/raw-body/node_modules/http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "dev": true, "dependencies": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" }, - "engines": { - "node": ">= 0.8" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/raw-body/node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true - }, - "node_modules/raw-body/node_modules/statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", - "dev": true, "engines": { "node": ">= 0.8" } @@ -22784,23 +22720,23 @@ "dev": true }, "body-parser": { - "version": "1.20.5", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", - "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", + "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", "dev": true, "requires": { - "bytes": "~3.1.2", + "bytes": "3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", - "destroy": "~1.2.0", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "on-finished": "~2.4.1", - "qs": "~6.15.1", - "raw-body": "~2.5.3", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.13.0", + "raw-body": "2.5.2", "type-is": "~1.6.18", - "unpipe": "~1.0.0" + "unpipe": "1.0.0" }, "dependencies": { "depd": { @@ -22808,31 +22744,6 @@ "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", "dev": true - }, - "http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "dev": true, - "requires": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - } - }, - "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true - }, - "statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", - "dev": true } } }, @@ -25497,14 +25408,14 @@ } }, "express": { - "version": "4.22.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", - "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", "dev": true, "requires": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "~1.20.5", + "body-parser": "~1.20.3", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", @@ -25523,7 +25434,7 @@ "parseurl": "~1.3.3", "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", - "qs": "~6.15.1", + "qs": "~6.14.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", @@ -25559,6 +25470,15 @@ "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "dev": true }, + "qs": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "dev": true, + "requires": { + "side-channel": "^1.1.0" + } + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -30727,12 +30647,12 @@ } }, "qs": { - "version": "6.15.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", - "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", "dev": true, "requires": { - "side-channel": "^1.1.0" + "side-channel": "^1.0.6" } }, "query-string": { @@ -30811,48 +30731,15 @@ "dev": true }, "raw-body": { - "version": "2.5.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", - "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "dev": true, "requires": { - "bytes": "~3.1.2", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "unpipe": "~1.0.0" - }, - "dependencies": { - "depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "dev": true - }, - "http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "dev": true, - "requires": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - } - }, - "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true - }, - "statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", - "dev": true - } + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" } }, "react": { diff --git a/vlm/match.py b/vlm/match.py index 35f966374f..0a9a7dc743 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -2,7 +2,6 @@ import os from pyliftover.liftover import LiftOver import re -from typing import Optional, Tuple from vlm.clickhouse_utils import get_clickhouse_variant_counts @@ -43,7 +42,7 @@ '20', '21', '22', 'X', 'Y', 'M', } MIN_POS = 1 -MAX_POS = 300_000_000 +MAX_POS = 3e8 def get_variant_match(query: dict) -> dict: @@ -59,7 +58,14 @@ def get_variant_match(query: dict) -> dict: return _format_results(ac+lift_ac, hom+lift_hom, url) -def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str, liftover: Optional[Tuple[str, int, str]]) -> str: +def _liftover_variant(chrom: str, pos: int, genome_build: str) -> tuple[str, int, str]: + liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 + lo = LiftOver(f'{LIFTOVER_DIR}/{genome_build.lower()}_to_{liftover_genome_build.lower()}.over.chain.gz') + lifted_coord = lo.convert_coordinate(f'chr{chrom}', pos) + return (lifted_coord[0][0].replace('chr', ''), lifted_coord[0][1], liftover_genome_build) if lifted_coord and lifted_coord[0] else None + + +def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str, liftover: tuple[str, int, str]) -> str: if not SEQR_BASE_URL: return SEQR_BASE_URL @@ -130,19 +136,3 @@ def _format_results(ac: int, hom: int, url: str) -> dict: ] } } - - -LIFTOVERS = { - GENOME_VERSION_GRCh38: None, - GENOME_VERSION_GRCh37: None, -} - - -def _liftover_variant(chrom: str, pos: int, genome_build: str) -> Optional[Tuple[str, int, str]]: - liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 - if not LIFTOVERS[genome_build]: - LIFTOVERS[genome_build] = LiftOver( - f'{LIFTOVER_DIR}/{genome_build.lower()}_to_{liftover_genome_build.lower()}.over.chain.gz' - ) - lifted_coord = LIFTOVERS[genome_build].convert_coordinate(f'chr{chrom}', pos) - return (lifted_coord[0][0].replace('chr', ''), lifted_coord[0][1], liftover_genome_build) if lifted_coord and lifted_coord[0] else None diff --git a/vlm/requirements-test.txt b/vlm/requirements-test.txt index 0ae9f4ebc9..d459ee0e98 100644 --- a/vlm/requirements-test.txt +++ b/vlm/requirements-test.txt @@ -14,7 +14,7 @@ aiohttp==3.13.5 # aioresponses # pytest-aiohttp aioresponses==0.7.8 - # via -r vlm/requirements-test.in + # via -r vlm/requirements.in aiosignal==1.4.0 # via # -c vlm/requirements.txt @@ -24,7 +24,7 @@ attrs==26.1.0 # -c vlm/requirements.txt # aiohttp coverage==5.1 - # via -r vlm/requirements-test.in + # via -r vlm/requirements.in frozenlist==1.8.0 # via # -c vlm/requirements.txt @@ -59,7 +59,7 @@ pytest==9.0.3 # pytest-aiohttp # pytest-asyncio pytest-aiohttp==1.1.0 - # via -r vlm/requirements-test.in + # via -r vlm/requirements.in pytest-asyncio==1.3.0 # via pytest-aiohttp typing-extensions==4.15.0 From 1b29a7591bde6b2c8a53758b38ce1be4e1f1ff48 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 11:23:52 -0400 Subject: [PATCH 137/173] settings cleanup --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 3c10b95629..2ffdaf30c1 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -80,7 +80,7 @@ jobs: docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, enable_join_runtime_filters=0 TO vlm_test_user, clickhouse_test_user" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1 TO vlm_test_user, clickhouse_test_user" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - name: Set up Clickhouse Test Database From 83376dc58f38c32b2956c0b486eed831bbb85bbc Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 11:29:18 -0400 Subject: [PATCH 138/173] restore merge issues --- ui/package-lock.json | 269 +++++++++++++++++++++++++++----------- vlm/match.py | 28 ++-- vlm/requirements-test.txt | 6 +- 3 files changed, 213 insertions(+), 90 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index b487378b97..94c0415ba7 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -4707,23 +4707,23 @@ } }, "node_modules/body-parser": { - "version": "1.20.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", - "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "version": "1.20.5", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", + "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", "dev": true, "dependencies": { - "bytes": "3.1.2", + "bytes": "~3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.13.0", - "raw-body": "2.5.2", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.15.1", + "raw-body": "~2.5.3", "type-is": "~1.6.18", - "unpipe": "1.0.0" + "unpipe": "~1.0.0" }, "engines": { "node": ">= 0.8", @@ -4739,6 +4739,41 @@ "node": ">= 0.8" } }, + "node_modules/body-parser/node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/body-parser/node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/bonjour-service": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/bonjour-service/-/bonjour-service-1.3.0.tgz", @@ -7996,14 +8031,14 @@ } }, "node_modules/express": { - "version": "4.22.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", - "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "version": "4.22.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", + "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", "dev": true, "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "~1.20.3", + "body-parser": "~1.20.5", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", @@ -8022,7 +8057,7 @@ "parseurl": "~1.3.3", "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", - "qs": "~6.14.0", + "qs": "~6.15.1", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", @@ -8071,21 +8106,6 @@ "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "dev": true }, - "node_modules/express/node_modules/qs": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", - "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", - "dev": true, - "dependencies": { - "side-channel": "^1.1.0" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/express/node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -14543,12 +14563,12 @@ } }, "node_modules/qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "version": "6.15.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", + "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", "dev": true, "dependencies": { - "side-channel": "^1.0.6" + "side-channel": "^1.1.0" }, "engines": { "node": ">=0.6" @@ -14645,20 +14665,64 @@ } }, "node_modules/raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", "dev": true, "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" }, "engines": { "node": ">= 0.8" } }, + "node_modules/raw-body/node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/raw-body/node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body/node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/raw-body/node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/react": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react/-/react-17.0.2.tgz", @@ -22720,23 +22784,23 @@ "dev": true }, "body-parser": { - "version": "1.20.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", - "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "version": "1.20.5", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", + "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", "dev": true, "requires": { - "bytes": "3.1.2", + "bytes": "~3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.13.0", - "raw-body": "2.5.2", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.15.1", + "raw-body": "~2.5.3", "type-is": "~1.6.18", - "unpipe": "1.0.0" + "unpipe": "~1.0.0" }, "dependencies": { "depd": { @@ -22744,6 +22808,31 @@ "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", "dev": true + }, + "http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, + "requires": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true } } }, @@ -25408,14 +25497,14 @@ } }, "express": { - "version": "4.22.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", - "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "version": "4.22.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", + "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", "dev": true, "requires": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "~1.20.3", + "body-parser": "~1.20.5", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", @@ -25434,7 +25523,7 @@ "parseurl": "~1.3.3", "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", - "qs": "~6.14.0", + "qs": "~6.15.1", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", @@ -25470,15 +25559,6 @@ "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "dev": true }, - "qs": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", - "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", - "dev": true, - "requires": { - "side-channel": "^1.1.0" - } - }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -30647,12 +30727,12 @@ } }, "qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "version": "6.15.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", + "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", "dev": true, "requires": { - "side-channel": "^1.0.6" + "side-channel": "^1.1.0" } }, "query-string": { @@ -30731,15 +30811,48 @@ "dev": true }, "raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", "dev": true, "requires": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "dependencies": { + "depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true + }, + "http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, + "requires": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true + } } }, "react": { diff --git a/vlm/match.py b/vlm/match.py index 0a9a7dc743..35f966374f 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -2,6 +2,7 @@ import os from pyliftover.liftover import LiftOver import re +from typing import Optional, Tuple from vlm.clickhouse_utils import get_clickhouse_variant_counts @@ -42,7 +43,7 @@ '20', '21', '22', 'X', 'Y', 'M', } MIN_POS = 1 -MAX_POS = 3e8 +MAX_POS = 300_000_000 def get_variant_match(query: dict) -> dict: @@ -58,14 +59,7 @@ def get_variant_match(query: dict) -> dict: return _format_results(ac+lift_ac, hom+lift_hom, url) -def _liftover_variant(chrom: str, pos: int, genome_build: str) -> tuple[str, int, str]: - liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 - lo = LiftOver(f'{LIFTOVER_DIR}/{genome_build.lower()}_to_{liftover_genome_build.lower()}.over.chain.gz') - lifted_coord = lo.convert_coordinate(f'chr{chrom}', pos) - return (lifted_coord[0][0].replace('chr', ''), lifted_coord[0][1], liftover_genome_build) if lifted_coord and lifted_coord[0] else None - - -def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str, liftover: tuple[str, int, str]) -> str: +def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str, liftover: Optional[Tuple[str, int, str]]) -> str: if not SEQR_BASE_URL: return SEQR_BASE_URL @@ -136,3 +130,19 @@ def _format_results(ac: int, hom: int, url: str) -> dict: ] } } + + +LIFTOVERS = { + GENOME_VERSION_GRCh38: None, + GENOME_VERSION_GRCh37: None, +} + + +def _liftover_variant(chrom: str, pos: int, genome_build: str) -> Optional[Tuple[str, int, str]]: + liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 + if not LIFTOVERS[genome_build]: + LIFTOVERS[genome_build] = LiftOver( + f'{LIFTOVER_DIR}/{genome_build.lower()}_to_{liftover_genome_build.lower()}.over.chain.gz' + ) + lifted_coord = LIFTOVERS[genome_build].convert_coordinate(f'chr{chrom}', pos) + return (lifted_coord[0][0].replace('chr', ''), lifted_coord[0][1], liftover_genome_build) if lifted_coord and lifted_coord[0] else None diff --git a/vlm/requirements-test.txt b/vlm/requirements-test.txt index d459ee0e98..0ae9f4ebc9 100644 --- a/vlm/requirements-test.txt +++ b/vlm/requirements-test.txt @@ -14,7 +14,7 @@ aiohttp==3.13.5 # aioresponses # pytest-aiohttp aioresponses==0.7.8 - # via -r vlm/requirements.in + # via -r vlm/requirements-test.in aiosignal==1.4.0 # via # -c vlm/requirements.txt @@ -24,7 +24,7 @@ attrs==26.1.0 # -c vlm/requirements.txt # aiohttp coverage==5.1 - # via -r vlm/requirements.in + # via -r vlm/requirements-test.in frozenlist==1.8.0 # via # -c vlm/requirements.txt @@ -59,7 +59,7 @@ pytest==9.0.3 # pytest-aiohttp # pytest-asyncio pytest-aiohttp==1.1.0 - # via -r vlm/requirements.in + # via -r vlm/requirements-test.in pytest-asyncio==1.3.0 # via pytest-aiohttp typing-extensions==4.15.0 From 0434fcf5dfb9e621cf9286732c77ef559246b693 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 11:30:42 -0400 Subject: [PATCH 139/173] pip cache --- .github/workflows/vlm-unit-tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 2ffdaf30c1..9892a7f8b6 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -57,9 +57,10 @@ jobs: steps: - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.11 + cache: 'pip' - uses: actions/checkout@v2 - name: Install dependencies run: | From 4c26cdc5c99198f3d62d7bb0ec0b0e61cbf80395 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 11:35:00 -0400 Subject: [PATCH 140/173] pip cache order --- .github/workflows/vlm-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 9892a7f8b6..8a098f0b34 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -56,12 +56,12 @@ jobs: POSTGRES_PASSWORD: pgtest steps: + - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v5 with: python-version: 3.11 cache: 'pip' - - uses: actions/checkout@v2 - name: Install dependencies run: | python3 -m pip install --upgrade pip wheel From 0508de1f3860be3518ab74f55ed0d9e807c557c6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 11:39:02 -0400 Subject: [PATCH 141/173] remove unneeded fixture --- vlm/conftest.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/vlm/conftest.py b/vlm/conftest.py index 1c610862ba..ce1361140e 100644 --- a/vlm/conftest.py +++ b/vlm/conftest.py @@ -1,8 +1,5 @@ import pytest -@pytest.fixture(autouse=True) -def enable_db_access_for_all_tests(db): - pass @pytest.fixture(scope='session') def django_db_setup(request, django_db_blocker,django_db_keepdb): @@ -27,4 +24,4 @@ def django_db_setup(request, django_db_blocker,django_db_keepdb): except Exception as exc: # noqa: BLE001 request.node.warn( pytest.PytestWarning(f"Error when trying to teardown test databases: {exc!r}") - ) \ No newline at end of file + ) From 5a2365b7bc5144b00e15848aab252f49c3ed36e1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 11:52:49 -0400 Subject: [PATCH 142/173] verride method --- vlm/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vlm/conftest.py b/vlm/conftest.py index ce1361140e..9d160c10d2 100644 --- a/vlm/conftest.py +++ b/vlm/conftest.py @@ -1,5 +1,8 @@ import pytest +@pytest.fixture(scope='session') +def django_db_blocker(django_db_blocker, db): + pass @pytest.fixture(scope='session') def django_db_setup(request, django_db_blocker,django_db_keepdb): From 7005f0f5fa1504163a978ff1c9545de5a0fee206 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 12:07:43 -0400 Subject: [PATCH 143/173] override method --- vlm/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vlm/conftest.py b/vlm/conftest.py index 9d160c10d2..4b70caf2bb 100644 --- a/vlm/conftest.py +++ b/vlm/conftest.py @@ -1,7 +1,7 @@ import pytest -@pytest.fixture(scope='session') -def django_db_blocker(django_db_blocker, db): +@pytest.fixture +def _django_db_marker(_django_db_marker, db): pass @pytest.fixture(scope='session') From 759df9c048cd6c90b1be04ef02d66f7ed57f8e5b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 12:21:08 -0400 Subject: [PATCH 144/173] update test --- vlm/test_vlm.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/vlm/test_vlm.py b/vlm/test_vlm.py index 02134c5e70..87f24f621e 100644 --- a/vlm/test_vlm.py +++ b/vlm/test_vlm.py @@ -42,7 +42,8 @@ async def test_match(self, mocked_responses): jwt_body = {'iss': 'https://vlm-auth.us.auth0.com/', 'azp': REQUESTER_CLIENT_ID} headers = {'Authorization': f'Bearer {jwt.encode(jwt_body, "")}'} - async with self.client.request('GET', '/vlm/match?assemblyId=GRCh38&referenceName=1&start=38724419&referenceBases=T&alternateBases=G', headers=headers) as resp: + self.maxDiff = None + async with self.client.request('GET', '/vlm/match?assemblyId=GRCh38&referenceName=1&start=10439&referenceBases=AC&alternateBases=A', headers=headers) as resp: self.assertEqual(resp.status, 200) resp_json = await resp.json() self.assertDictEqual(resp_json, { @@ -52,7 +53,7 @@ async def test_match(self, mocked_responses): 'id': 'TestVLM', 'label': 'TestVLM browser', }, - 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=38&variantId=1-38724419-T-G', + 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=38&variantId=1-10439-AC-A', 'email': None, } ], @@ -76,14 +77,14 @@ async def test_match(self, mocked_responses): 'exists': True, 'id': 'TestVLM Homozygous', 'results': [], - 'resultsCount': 7, + 'resultsCount': 3, 'setType': 'genomicVariant' }, { 'exists': True, 'id': 'TestVLM Heterozygous', 'results': [], - 'resultsCount': 23, + 'resultsCount': 2, 'setType': 'genomicVariant' }, { @@ -113,7 +114,7 @@ async def test_match(self, mocked_responses): headers={'Authorization': 'Bearer test_token'}, ) self.assertIn( - 'Received match request from Test Node: assemblyId=GRCh38&referenceName=1&start=38724419&referenceBases=T&alternateBases=G', + 'Received match request from Test Node: assemblyId=GRCh38&referenceName=1&start=10439&referenceBases=AC&alternateBases=A', self._caplog.messages, ) From 0ca748fda9f1266b836d16f63bb996e435beb541 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 12:24:44 -0400 Subject: [PATCH 145/173] reload gt stats --- vlm/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vlm/conftest.py b/vlm/conftest.py index 4b70caf2bb..449e667e14 100644 --- a/vlm/conftest.py +++ b/vlm/conftest.py @@ -8,6 +8,8 @@ def _django_db_marker(_django_db_marker, db): def django_db_setup(request, django_db_blocker,django_db_keepdb): from django.core.management import call_command from django.test.utils import setup_databases, teardown_databases + from clickhouse_search.models.gt_stats_models import ProjectsToGtStatsGRCh37SnvIndel, ProjectsToGtStatsSnvIndel, \ + GtStatsDictGRCh37SnvIndel, GtStatsDictSnvIndel with django_db_blocker.unblock(): db_cfg = setup_databases( @@ -17,6 +19,10 @@ def django_db_setup(request, django_db_blocker,django_db_keepdb): keepdb=django_db_keepdb, ) call_command('loaddata', 'clickhouse_search', '--database=clickhouse_write') + ProjectsToGtStatsGRCh37SnvIndel.refresh() + ProjectsToGtStatsSnvIndel.refresh() + GtStatsDictGRCh37SnvIndel.reload() + GtStatsDictSnvIndel.reload() yield From 0cb106c779304c5fc61f6fec7a58af8992af6b39 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 12:37:06 -0400 Subject: [PATCH 146/173] correct counts --- vlm/test_vlm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vlm/test_vlm.py b/vlm/test_vlm.py index 87f24f621e..d9ef2e1ab6 100644 --- a/vlm/test_vlm.py +++ b/vlm/test_vlm.py @@ -43,7 +43,7 @@ async def test_match(self, mocked_responses): headers = {'Authorization': f'Bearer {jwt.encode(jwt_body, "")}'} self.maxDiff = None - async with self.client.request('GET', '/vlm/match?assemblyId=GRCh38&referenceName=1&start=10439&referenceBases=AC&alternateBases=A', headers=headers) as resp: + async with self.client.request('GET', '/vlm/match?assemblyId=GRCh38&referenceName=1&start=38724419&referenceBases=T&alternateBases=G', headers=headers) as resp: self.assertEqual(resp.status, 200) resp_json = await resp.json() self.assertDictEqual(resp_json, { @@ -53,7 +53,7 @@ async def test_match(self, mocked_responses): 'id': 'TestVLM', 'label': 'TestVLM browser', }, - 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=38&variantId=1-10439-AC-A', + 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=38&variantId=1-38724419-T-G', 'email': None, } ], @@ -84,7 +84,7 @@ async def test_match(self, mocked_responses): 'exists': True, 'id': 'TestVLM Heterozygous', 'results': [], - 'resultsCount': 2, + 'resultsCount': 4, 'setType': 'genomicVariant' }, { @@ -114,7 +114,7 @@ async def test_match(self, mocked_responses): headers={'Authorization': 'Bearer test_token'}, ) self.assertIn( - 'Received match request from Test Node: assemblyId=GRCh38&referenceName=1&start=10439&referenceBases=AC&alternateBases=A', + 'Received match request from Test Node: assemblyId=GRCh38&referenceName=1&start=38724419&referenceBases=T&alternateBases=G', self._caplog.messages, ) From 3c28d813766874bfaff940403dc1a9a242628b4c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 12:38:26 -0400 Subject: [PATCH 147/173] correct counts --- vlm/test_vlm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vlm/test_vlm.py b/vlm/test_vlm.py index d9ef2e1ab6..f69a7ead2c 100644 --- a/vlm/test_vlm.py +++ b/vlm/test_vlm.py @@ -149,14 +149,14 @@ async def test_match(self, mocked_responses): 'exists': True, 'id': 'TestVLM Homozygous', 'results': [], - 'resultsCount': 1508, + 'resultsCount': 2, 'setType': 'genomicVariant' }, { 'exists': True, 'id': 'TestVLM Heterozygous', 'results': [], - 'resultsCount': 1695, + 'resultsCount': 0, 'setType': 'genomicVariant' }, { From 012ad43b2a1bdc01f0d69ca737b70d5b3c788795 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 12:59:49 -0400 Subject: [PATCH 148/173] update total counts --- vlm/test_vlm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vlm/test_vlm.py b/vlm/test_vlm.py index f69a7ead2c..9dbab9cb56 100644 --- a/vlm/test_vlm.py +++ b/vlm/test_vlm.py @@ -69,7 +69,7 @@ async def test_match(self, mocked_responses): }, 'responseSummary': { 'exists': True, - 'total': 30, + 'total': 7, }, 'response': { 'resultSets': [ @@ -141,7 +141,7 @@ async def test_match(self, mocked_responses): }, 'responseSummary': { 'exists': True, - 'total': 3203, + 'total': 2, }, 'response': { 'resultSets': [ From bf26cf50c8f9cc75f0fcfef19d1965721470d9ed Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 13:13:09 -0400 Subject: [PATCH 149/173] fix counts --- vlm/test_vlm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vlm/test_vlm.py b/vlm/test_vlm.py index 9dbab9cb56..366015b187 100644 --- a/vlm/test_vlm.py +++ b/vlm/test_vlm.py @@ -141,7 +141,7 @@ async def test_match(self, mocked_responses): }, 'responseSummary': { 'exists': True, - 'total': 2, + 'total': 1, }, 'response': { 'resultSets': [ @@ -149,11 +149,11 @@ async def test_match(self, mocked_responses): 'exists': True, 'id': 'TestVLM Homozygous', 'results': [], - 'resultsCount': 2, + 'resultsCount': 1, 'setType': 'genomicVariant' }, { - 'exists': True, + 'exists': False, 'id': 'TestVLM Heterozygous', 'results': [], 'resultsCount': 0, From 929f6a48b5fe10a3f53d93f15ff8322a04cbc864 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 13:21:15 -0400 Subject: [PATCH 150/173] update coverage --- .github/workflows/vlm-unit-tests.yaml | 2 +- vlm/test_vlm.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 8a098f0b34..c8cf522db2 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -114,6 +114,6 @@ jobs: export CLICKHOUSE_VLM_USERNAME=vlm_test_user export CLICKHOUSE_VLM_PASSWORD=vlm_test_password export CLICKHOUSE_DATABASE=test_seqr - coverage run --source="./vlm" --omit="./vlm/__main__.py" -m pytest vlm/ + coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/conftest.py" -m pytest vlm/ coverage report -m --fail-under=95 diff --git a/vlm/test_vlm.py b/vlm/test_vlm.py index 366015b187..c048c0898e 100644 --- a/vlm/test_vlm.py +++ b/vlm/test_vlm.py @@ -42,7 +42,6 @@ async def test_match(self, mocked_responses): jwt_body = {'iss': 'https://vlm-auth.us.auth0.com/', 'azp': REQUESTER_CLIENT_ID} headers = {'Authorization': f'Bearer {jwt.encode(jwt_body, "")}'} - self.maxDiff = None async with self.client.request('GET', '/vlm/match?assemblyId=GRCh38&referenceName=1&start=38724419&referenceBases=T&alternateBases=G', headers=headers) as resp: self.assertEqual(resp.status, 200) resp_json = await resp.json() From 394e7c0d19fa7f0dc95aee588f2119d8c2f0b558 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 14:38:18 -0400 Subject: [PATCH 151/173] fix vlm requirements install --- vlm/deploy/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vlm/deploy/Dockerfile b/vlm/deploy/Dockerfile index 33a0fce9e5..51901775cc 100644 --- a/vlm/deploy/Dockerfile +++ b/vlm/deploy/Dockerfile @@ -2,13 +2,13 @@ FROM python:3.11-slim-bullseye LABEL maintainer="Broad TGG" -RUN pip install --no-cache-dir -r vlm/requirements.txt - WORKDIR /vlm # Application Code COPY vlm/ . +RUN pip install --no-cache-dir -r requirements.txt + WORKDIR / EXPOSE 7000 CMD ["python3", "-m", "vlm"] From 583b627b3e1b63f0805af864efff389deaedd2f0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 14:56:25 -0400 Subject: [PATCH 152/173] disable rna reference tracks for grach37 --- .../components/panel/family/FamilyReads.jsx | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/ui/shared/components/panel/family/FamilyReads.jsx b/ui/shared/components/panel/family/FamilyReads.jsx index 70d6a41a55..bd95805019 100644 --- a/ui/shared/components/panel/family/FamilyReads.jsx +++ b/ui/shared/components/panel/family/FamilyReads.jsx @@ -296,7 +296,13 @@ class FamilyReads extends React.PureComponent { } updateReads = (familyGuid, locus, sampleTypes, tissueType) => { - this.setState({ openFamily: familyGuid, sampleTypes, locus, rnaReferences: TISSUE_REFERENCES_LOOKUP[tissueType] }) + const project = familyGuid && this.getProjectForFamily(familyGuid) + this.setState({ + openFamily: familyGuid, + sampleTypes, + locus, + rnaReferences: project?.genomeVersion === GENOME_VERSION_38 ? TISSUE_REFERENCES_LOOKUP[tissueType] : [], + }) } getProjectForFamily = (familyGuid) => { @@ -414,6 +420,7 @@ class FamilyReads extends React.PureComponent { const dnaTrackOptions = DNA_TRACK_TYPE_OPTIONS.filter(({ value }) => igvSampleIndividuals[value]) const rnaTrackOptions = RNA_TRACK_TYPE_OPTIONS.filter(({ value }) => igvSampleIndividuals[value]) const project = openFamily && this.getProjectForFamily(openFamily) + const isGrch38 = project?.genomeVersion === GENOME_VERSION_38 const geneLocus = project && variant && getGeneLocus(variant, genesById, project) const locusOptions = [ { text: 'Variant', value: geneLocus && getVariantLocus(variant) }, @@ -463,15 +470,17 @@ class FamilyReads extends React.PureComponent {
{sampleTypes.some(sampleType => RNA_TRACK_TYPE_LOOKUP.has(sampleType)) && (
- Reference Tracks - {this.gtexSelector('Normalized', NORM_GTEX_TRACK_OPTIONS)} - {this.gtexSelector('Aggregate', AGG_GTEX_TRACK_OPTIONS)} - + {isGrch38 && Reference Tracks} + {isGrch38 && this.gtexSelector('Normalized', NORM_GTEX_TRACK_OPTIONS)} + {isGrch38 && this.gtexSelector('Aggregate', AGG_GTEX_TRACK_OPTIONS)} + {isGrch38 && ( + + )} Junction Filters Date: Fri, 29 May 2026 16:13:58 -0400 Subject: [PATCH 153/173] update gnomad constartint citation --- ui/shared/utils/constants.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index 1b8c22e55c..b7511624bf 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -1574,7 +1574,11 @@ export const ORDERED_PREDICTOR_FIELDS = [ fieldTitle: 'gnomAD Constraint', displayOnly: true, thresholds: [undefined, undefined, 2.18, 4, undefined], - citation: CLINGEN_CITATION, + citation: { + name: '2022 recommendations from the gnomAD team', + linkText: 'Blog', + href: 'https://gnomad.broadinstitute.org/news/2022-10-the-addition-of-a-genomic-constraint-metric-to-gnomad/', + }, }, { field: 'haplogroup_defining', indicatorMap: { true: { color: 'green', value: '' } } }, { field: 'mitotip', indicatorMap: MITOTIP_MAP, fieldTitle: 'MitoTIP' }, @@ -1645,8 +1649,10 @@ export const predictorColorRanges = (thresholds, { citation, reverseThresholds, })} {citation && ( - {`Based on ${citation.name} (PMID: `} - {citation.pmid} + {`Based on ${citation.name} (${citation.pmid ? 'PMID: ' : ''}`} + + {citation.pmid || citation.linkText} + ) )} From 71d86087e632a1bdbf55df7669551bde72dcdaeb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 29 May 2026 16:14:48 -0400 Subject: [PATCH 154/173] fix linting --- ui/shared/utils/constants.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index b7511624bf..4580766e59 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -1650,7 +1650,7 @@ export const predictorColorRanges = (thresholds, { citation, reverseThresholds, {citation && ( {`Based on ${citation.name} (${citation.pmid ? 'PMID: ' : ''}`} - + {citation.pmid || citation.linkText} ) From 227b2a322c18be5f6a2beec6a62e35cf71fc50c0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 3 Jun 2026 16:19:24 -0400 Subject: [PATCH 155/173] sync tag updates to postgres dicts --- .../commands/update_all_reference_data.py | 3 +++ seqr/views/apis/saved_variant_api.py | 24 +++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/reference_data/management/commands/update_all_reference_data.py b/reference_data/management/commands/update_all_reference_data.py index efbc9265ee..9a4f9cbe17 100644 --- a/reference_data/management/commands/update_all_reference_data.py +++ b/reference_data/management/commands/update_all_reference_data.py @@ -2,6 +2,7 @@ from collections import OrderedDict from django.core.management.base import BaseCommand, CommandError +from clickhouse_search.models.postgres_dicts import GeneIdDict from panelapp.models import PanelAppAU, PanelAppUK from reference_data.utils.gene_utils import get_genes_by_id_and_symbol from reference_data.models import GeneInfo, TranscriptInfo, HumanPhenotypeOntology, RefseqTranscript, GeneConstraint, \ @@ -57,6 +58,8 @@ def handle(self, *args, **options): data_model_name = GeneInfo.__name__ self._update_gencode(current_versions.get(data_model_name), options['gene_symbol_change_dir']) self._track_success_updates(data_model_name, latest_version, current_versions, updated) + GeneIdDict.reload() + gene_ids_to_gene, gene_symbols_to_gene = get_genes_by_id_and_symbol() if to_update else (None, None) for data_cls, latest_version in to_update.items(): diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 099d210914..8135c08f77 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -3,6 +3,7 @@ from django.db.models import Q +from clickhouse_search.models.postgres_dicts import DiscoveryVariantDict, ExcludedVariantDict from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\ Family, GeneNote, Project, Dataset from seqr.utils.xpos_utils import get_xpos @@ -13,7 +14,7 @@ get_json_for_saved_variants_child_entities, get_json_for_gene_notes_by_gene_id, STRUCTURED_METADATA_TAG_TYPES from seqr.views.utils.permissions_utils import get_project_and_check_permissions, check_project_permissions, \ login_and_policies_required -from seqr.views.utils.variant_utils import get_variants_response, parse_saved_variant_json +from seqr.views.utils.variant_utils import get_variants_response, parse_saved_variant_json, DISCOVERY_CATEGORY logger = logging.getLogger(__name__) @@ -274,9 +275,9 @@ def _update_variant_tag_models(request, variant_guids, tag_key, model_cls, get_t tag_type = tag_key.lower().rstrip('s') updated_data = request_json.get(tag_key, []) - deleted_guids = _delete_removed_tags(saved_variants, all_variant_guids, updated_data, request.user, tag_type, protected_tag_types) + deleted_guids, has_discovery_update, has_excluded_update = _delete_removed_tags(saved_variants, all_variant_guids, updated_data, request.user, tag_type, protected_tag_types) - _update_tags(saved_variants, request_json, request.user, tag_key, model_cls, get_tag_create_data) + _update_tags(saved_variants, request_json, request.user, tag_key, model_cls, get_tag_create_data, has_discovery_update=has_discovery_update, has_excluded_update=has_excluded_update) saved_variant_id_map = {sv.id: sv.guid for sv in saved_variants} tags, variant_tag_map = get_json_for_saved_variants_child_entities(model_cls, saved_variant_id_map) @@ -311,15 +312,18 @@ def _delete_removed_tags(saved_variants, all_variant_guids, tag_updates, user, t remove_tags = tag_set.exclude(guid__in=existing_tag_guids) if protected_tag_types: remove_tags = remove_tags.exclude(variant_tag_type__name__in=protected_tag_types) + track_updates = tag_type == 'tag' + has_discovery_update = remove_tags.filter(variant_tag_type__category=DISCOVERY_CATEGORY).exists() if track_updates else False + has_excluded_update = remove_tags.filter(variant_tag_type__name='Excluded').exists() if track_updates else False for tag in remove_tags: tag_variant_guids = {sv.guid for sv in tag.saved_variants.all()} if tag_variant_guids == all_variant_guids: deleted_tag_guids.append(tag.guid) tag.delete_model(user, user_can_delete=True) - return deleted_tag_guids + return deleted_tag_guids, has_discovery_update, has_excluded_update -def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=VariantTag, get_tag_create_data=_get_tag_type_create_data): +def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=VariantTag, get_tag_create_data=_get_tag_type_create_data, has_discovery_update=False, has_excluded_update=False): tags = tags_json.get(tag_key, []) for tag in tags: if tag.get('tagGuid'): @@ -327,6 +331,11 @@ def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=Vari update_model_from_json(model, tag, user=user, allow_unknown_keys=True) else: create_data = get_tag_create_data(tag, saved_variants=saved_variants) + if 'variant_tag_type' in create_data: + if create_data['variant_tag_type'].category == DISCOVERY_CATEGORY: + has_discovery_update = True + elif create_data['variant_tag_type'].name == 'Excluded': + has_excluded_update = True create_data.update({ 'metadata': tag.get('metadata'), 'search_hash': tags_json.get('searchHash'), @@ -334,6 +343,11 @@ def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=Vari model = create_model_from_json(model_cls, create_data, user) model.saved_variants.set(saved_variants) + if has_discovery_update: + DiscoveryVariantDict.reload() + if has_excluded_update: + ExcludedVariantDict.reload() + @login_and_policies_required def update_variant_main_transcript(request, variant_guid, transcript_id): From 0498c89050f553e4bc739ded557d7ce3dab6c6e6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 3 Jun 2026 16:30:56 -0400 Subject: [PATCH 156/173] reload metadata on project update --- seqr/views/apis/project_api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/seqr/views/apis/project_api.py b/seqr/views/apis/project_api.py index db0eb69245..c0bedce0b6 100644 --- a/seqr/views/apis/project_api.py +++ b/seqr/views/apis/project_api.py @@ -7,6 +7,7 @@ from django.utils import timezone from notifications.models import Notification +from clickhouse_search.models.postgres_dicts import IndividualMetadataDict from matchmaker.models import MatchmakerSubmission from seqr.models import Project, Family, Individual, Dataset, RnaSample, FamilyNote, PhenotypePrioritization, CAN_EDIT from seqr.utils.file_utils import file_iter @@ -59,6 +60,7 @@ def create_project_handler(request): project_args['is_mme_enabled'] = False project = create_model_from_json(Project, project_args, user=request.user) + IndividualMetadataDict.reload() return create_json_response({ 'projectsByGuid': { @@ -89,6 +91,8 @@ def update_project_handler(request, project_guid): updated_fields = {'consent_code'} update_project_from_json(project, request_json, request.user, allow_unknown_keys=True, updated_fields=updated_fields) + if 'restrict_sharing' in updated_fields or 'vlm_contact_email' in updated_fields: + IndividualMetadataDict.reload() return create_json_response({ 'projectsByGuid': { @@ -378,6 +382,8 @@ def _delete_project(project_guid, user): project.delete_model(user, user_can_delete=True) + IndividualMetadataDict.reload() + if anvil_enabled() and not is_internal_anvil_project(project): AirtableSession(user, base=AirtableSession.ANVIL_BASE).safe_patch_records( ANVIL_REQUEST_TRACKING_TABLE, From 1386778c25ca31efc0a2d622c5827439a2b954bc Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 3 Jun 2026 16:44:19 -0400 Subject: [PATCH 157/173] family update reload --- seqr/views/apis/family_api.py | 8 +++++++- seqr/views/utils/json_to_orm_utils.py | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index d3684a5ef6..1c0dbb9194 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -5,6 +5,7 @@ from django.db.models import Count, Max, Q, F from django.db.models.fields.files import ImageFieldFile +from clickhouse_search.models.postgres_dicts import IndividualMetadataDict from matchmaker.models import MatchmakerSubmission from reference_data.models import Omim from seqr.utils.gene_utils import get_genes_for_variant_display @@ -210,6 +211,8 @@ def delete_families_handler(request, project_guid): # delete families Family.bulk_delete(request.user, project=project, guid__in=family_guids_to_delete) + IndividualMetadataDict.reload() + # send response return create_json_response({ 'individualsByGuid': { @@ -230,9 +233,12 @@ def update_family_fields_handler(request, family_guid): request_json = json.loads(request.body) immutable_keys = [] if external_anvil_project_can_edit(family.project, request.user) else ['family_id'] + updated_fields = set() update_family_from_json(family, request_json, user=request.user, allow_unknown_keys=True, immutable_keys=[ 'display_name', - ] + immutable_keys) + ] + immutable_keys, updated_fields=updated_fields) + if updated_fields.intersection({'post_discovery_omim_numbers', 'post_discovery_mondo_id', 'analysis_status'}): + IndividualMetadataDict.reload() return create_json_response({ family.guid: _get_json_for_model(family, user=request.user, process_result=_set_display_name) diff --git a/seqr/views/utils/json_to_orm_utils.py b/seqr/views/utils/json_to_orm_utils.py index 02a65aa805..909c8b778b 100644 --- a/seqr/views/utils/json_to_orm_utils.py +++ b/seqr/views/utils/json_to_orm_utils.py @@ -15,14 +15,14 @@ def update_project_from_json(project, json, user, allow_unknown_keys=False, upda immutable_keys=['consent_code', 'genome_version', 'workspace_namespace', 'workspace_name']) -def update_family_from_json(family, json, user, allow_unknown_keys=False, immutable_keys=None): +def update_family_from_json(family, json, user, allow_unknown_keys=False, immutable_keys=None, updated_fields=None): if json.get('displayName') and json['displayName'] == family.family_id: json['displayName'] = '' immutable_keys = (immutable_keys or []) + ['pedigree_image', 'assigned_analyst', 'case_review_summary', 'case_review_notes', 'guid'] return update_model_from_json( - family, json, user=user, allow_unknown_keys=allow_unknown_keys, immutable_keys=immutable_keys, + family, json, user=user, allow_unknown_keys=allow_unknown_keys, immutable_keys=immutable_keys, updated_fields=updated_fields, ) From a5fabc3dc619d978339c94601a13637765fe383f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 4 Jun 2026 11:13:47 -0400 Subject: [PATCH 158/173] individual update reloads --- seqr/views/apis/individual_api.py | 8 ++++++++ seqr/views/utils/individual_utils.py | 17 +++++++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/seqr/views/apis/individual_api.py b/seqr/views/apis/individual_api.py index 06da8a78a8..a717116d22 100644 --- a/seqr/views/apis/individual_api.py +++ b/seqr/views/apis/individual_api.py @@ -8,6 +8,7 @@ from django.contrib.auth.models import User from django.db.models import prefetch_related_objects +from clickhouse_search.models.postgres_dicts import IndividualMetadataDict, SexDict, AffectedDict from reference_data.models import HumanPhenotypeOntology from seqr.models import Individual, Family, CAN_VIEW from seqr.utils.file_utils import file_iter @@ -93,6 +94,7 @@ def update_individual_hpo_terms(request, individual_guid): for key in feature_fields } update_individual_from_json(individual, update_json, user=request.user, allow_features_update=True) + IndividualMetadataDict.reload() individual_json = {k: getattr(individual, _to_snake_case(k)) for k in feature_fields} add_individual_hpo_details([individual_json]) @@ -218,6 +220,9 @@ def delete_individuals_handler(request, project_guid): # delete the individuals families_with_deleted_individuals = delete_individuals(project, individual_guids_to_delete, request.user) + AffectedDict.reload() + SexDict.reload() + IndividualMetadataDict.reload() deleted_individuals_by_guid = { individual_guid: None for individual_guid in individual_guids_to_delete @@ -665,6 +670,9 @@ def save_individuals_metadata_table_handler(request, project_guid, upload_file_i if record.get(ASSIGNED_ANALYST_COL): family_assigned_analysts[record[ASSIGNED_ANALYST_COL]].append(individual.family.id) + if any(FEATURES_COL in record for record in json_records): + IndividualMetadataDict.reload() + response = { 'individualsByGuid': { individual['individualGuid']: individual for individual in _get_json_for_individuals( diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index ea21b57f45..e396c365e9 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -1,6 +1,6 @@ from collections import defaultdict -from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict +from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict, IndividualMetadataDict from matchmaker.models import MatchmakerSubmission, MatchmakerResult from seqr.models import Dataset, IgvSample, RnaSample, Individual, Family, FamilyNote from seqr.utils.middleware import ErrorsWarningsException @@ -25,6 +25,7 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge updated_individuals = set() updated_affected = set() updated_sex = set() + updated_metadata = set() updated_note_ids = [] parent_updates = [] num_created_families = 0 @@ -56,7 +57,7 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge for record in individual_records: created_individual = _update_from_record( - record, user, families_by_id, individual_lookup, updated_family_ids, updated_individuals, updated_affected, updated_sex, parent_updates, updated_note_ids, allow_features_update) + record, user, families_by_id, individual_lookup, updated_family_ids, updated_individuals, updated_affected, updated_sex, updated_metadata, parent_updates, updated_note_ids, allow_features_update) if created_individual: num_created_individuals += 1 @@ -70,12 +71,14 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge updated_family_models = Family.objects.filter(id__in=updated_family_ids) _remove_pedigree_images(updated_family_models, user) - if updated_affected: + if updated_affected or num_created_individuals > 0: AffectedDict.reload(user) - if not skip_gt_stats_rebuild: + if updated_affected and not skip_gt_stats_rebuild: trigger_rebuild_gt_stats(project, user) - if updated_sex: + if updated_sex or num_created_individuals > 0: SexDict.reload(user) + if updated_metadata or num_created_families > 0 or num_created_individuals > 0: + IndividualMetadataDict.reload(user) pedigree_json = None if get_update_json: @@ -90,7 +93,7 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge return pedigree_json -def _update_from_record(record, user, families_by_id, individual_lookup, updated_family_ids, updated_individuals, updated_affected, updated_sex, parent_updates, updated_note_ids, allow_features_update): +def _update_from_record(record, user, families_by_id, individual_lookup, updated_family_ids, updated_individuals, updated_affected, updated_sex, updated_metadata, parent_updates, updated_note_ids, allow_features_update): family_id = _get_record_family_id(record) family = families_by_id.get(family_id) created_individual = False @@ -153,6 +156,8 @@ def _update_from_record(record, user, families_by_id, individual_lookup, updated updated_affected.add(individual) if 'sex' in updated_fields: updated_sex.add(individual) + if 'features' in updated_fields: + updated_metadata.add(individual) if family.pedigree_image: updated_family_ids.add(family.id) From 622fc4cd5752f4c9aba29f990eed5d9d46116f63 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 4 Jun 2026 11:16:48 -0400 Subject: [PATCH 159/173] log user with relaods --- seqr/views/apis/family_api.py | 4 ++-- seqr/views/apis/individual_api.py | 10 +++++----- seqr/views/apis/project_api.py | 10 ++++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 1c0dbb9194..dc4bde189c 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -211,7 +211,7 @@ def delete_families_handler(request, project_guid): # delete families Family.bulk_delete(request.user, project=project, guid__in=family_guids_to_delete) - IndividualMetadataDict.reload() + IndividualMetadataDict.reload(request.user) # send response return create_json_response({ @@ -238,7 +238,7 @@ def update_family_fields_handler(request, family_guid): 'display_name', ] + immutable_keys, updated_fields=updated_fields) if updated_fields.intersection({'post_discovery_omim_numbers', 'post_discovery_mondo_id', 'analysis_status'}): - IndividualMetadataDict.reload() + IndividualMetadataDict.reload(request.user) return create_json_response({ family.guid: _get_json_for_model(family, user=request.user, process_result=_set_display_name) diff --git a/seqr/views/apis/individual_api.py b/seqr/views/apis/individual_api.py index a717116d22..9d7a0697b8 100644 --- a/seqr/views/apis/individual_api.py +++ b/seqr/views/apis/individual_api.py @@ -94,7 +94,7 @@ def update_individual_hpo_terms(request, individual_guid): for key in feature_fields } update_individual_from_json(individual, update_json, user=request.user, allow_features_update=True) - IndividualMetadataDict.reload() + IndividualMetadataDict.reload(request.user) individual_json = {k: getattr(individual, _to_snake_case(k)) for k in feature_fields} add_individual_hpo_details([individual_json]) @@ -220,9 +220,9 @@ def delete_individuals_handler(request, project_guid): # delete the individuals families_with_deleted_individuals = delete_individuals(project, individual_guids_to_delete, request.user) - AffectedDict.reload() - SexDict.reload() - IndividualMetadataDict.reload() + AffectedDict.reload(request.user) + SexDict.reload(request.user) + IndividualMetadataDict.reload(request.user) deleted_individuals_by_guid = { individual_guid: None for individual_guid in individual_guids_to_delete @@ -671,7 +671,7 @@ def save_individuals_metadata_table_handler(request, project_guid, upload_file_i family_assigned_analysts[record[ASSIGNED_ANALYST_COL]].append(individual.family.id) if any(FEATURES_COL in record for record in json_records): - IndividualMetadataDict.reload() + IndividualMetadataDict.reload(request.user) response = { 'individualsByGuid': { diff --git a/seqr/views/apis/project_api.py b/seqr/views/apis/project_api.py index c0bedce0b6..591d1659cd 100644 --- a/seqr/views/apis/project_api.py +++ b/seqr/views/apis/project_api.py @@ -7,7 +7,7 @@ from django.utils import timezone from notifications.models import Notification -from clickhouse_search.models.postgres_dicts import IndividualMetadataDict +from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict, IndividualMetadataDict from matchmaker.models import MatchmakerSubmission from seqr.models import Project, Family, Individual, Dataset, RnaSample, FamilyNote, PhenotypePrioritization, CAN_EDIT from seqr.utils.file_utils import file_iter @@ -60,7 +60,7 @@ def create_project_handler(request): project_args['is_mme_enabled'] = False project = create_model_from_json(Project, project_args, user=request.user) - IndividualMetadataDict.reload() + IndividualMetadataDict.reload(request.user) return create_json_response({ 'projectsByGuid': { @@ -92,7 +92,7 @@ def update_project_handler(request, project_guid): update_project_from_json(project, request_json, request.user, allow_unknown_keys=True, updated_fields=updated_fields) if 'restrict_sharing' in updated_fields or 'vlm_contact_email' in updated_fields: - IndividualMetadataDict.reload() + IndividualMetadataDict.reload(request.user) return create_json_response({ 'projectsByGuid': { @@ -382,7 +382,9 @@ def _delete_project(project_guid, user): project.delete_model(user, user_can_delete=True) - IndividualMetadataDict.reload() + AffectedDict.reload(user) + SexDict.reload(user) + IndividualMetadataDict.reload(user) if anvil_enabled() and not is_internal_anvil_project(project): AirtableSession(user, base=AirtableSession.ANVIL_BASE).safe_patch_records( From 5722238b3f586d0dbab53a66476e36b74a19f6f7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 4 Jun 2026 11:39:56 -0400 Subject: [PATCH 160/173] fix tests --- .../management/tests/update_all_reference_data_tests.py | 2 ++ reference_data/management/tests/update_gencode_tests.py | 3 +++ seqr/views/apis/project_api.py | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/reference_data/management/tests/update_all_reference_data_tests.py b/reference_data/management/tests/update_all_reference_data_tests.py index 32b0c83779..5f1bb2b263 100644 --- a/reference_data/management/tests/update_all_reference_data_tests.py +++ b/reference_data/management/tests/update_all_reference_data_tests.py @@ -42,6 +42,7 @@ def _mock_handler(_cls, **kwargs): class NewDbUpdateAllReferenceDataTest(BaseUpdateAllReferenceDataTest): + databases = '__all__' fixtures = ['users'] def test_empty_db_update_all_reference_data_command(self): @@ -78,6 +79,7 @@ def test_empty_db_update_all_reference_data_command(self): self.mock_slack.assert_not_called() self.assert_json_logs(user=None, expected=[ + ('Reloading dictionary seqrdb_gene_ids', None), ('unable to update PrimateAI: Primate_AI failed', { 'severity': 'ERROR', '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', diff --git a/reference_data/management/tests/update_gencode_tests.py b/reference_data/management/tests/update_gencode_tests.py index e5b6ed3db1..82a1945508 100644 --- a/reference_data/management/tests/update_gencode_tests.py +++ b/reference_data/management/tests/update_gencode_tests.py @@ -54,6 +54,8 @@ class UpdateGencodeTest(ReferenceDataCommandTestCase): + databases = '__all__' + def setUp(self): super().setUp() @@ -202,6 +204,7 @@ def test_update_gencode_latest_command(self): ('Loaded 3 RefseqTranscript records', None), ('Skipped 1 records with unrecognized or duplicated transcripts', None), ('Updated GeneInfo reference data from version "31" to version "39"', None), + ('Reloading dictionary seqrdb_gene_ids', None), ('Done', None), ('Updated: GeneInfo', None), ]) diff --git a/seqr/views/apis/project_api.py b/seqr/views/apis/project_api.py index 591d1659cd..3585ae625e 100644 --- a/seqr/views/apis/project_api.py +++ b/seqr/views/apis/project_api.py @@ -82,7 +82,7 @@ def update_project_handler(request, project_guid): check_project_permissions(project, request.user, can_edit=True) request_json = json.loads(request.body) - updated_fields = None + updated_fields = set() consent_code = request_json.get('consentCode') if consent_code and consent_code != project.consent_code: if not user_is_pm(request.user): From 0b7806b271872be260824e3a3e255ed78ad4031c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 4 Jun 2026 12:04:26 -0400 Subject: [PATCH 161/173] fix saved variant data tests --- seqr/views/apis/saved_variant_api_tests.py | 134 ++++++++------------- 1 file changed, 47 insertions(+), 87 deletions(-) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 954ecb7d03..24046cab12 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -3,15 +3,15 @@ from django.urls.base import reverse -from clickhouse_search.models.reference_data_models import DbnsfpGRCh37SnvIndelMv, DbnsfpGRCh37SnvIndelDict +from clickhouse_search.all_search_tests import ClickhouseSearchTestCase from seqr.models import SavedVariant, VariantNote, VariantTag, VariantFunctionalData, Project from seqr.views.apis.saved_variant_api import saved_variant_data, create_variant_note_handler, create_saved_variant_handler, \ update_variant_note_handler, delete_variant_note_handler, update_variant_tags_handler, create_manual_saved_variant_handler, \ update_variant_main_transcript, update_variant_functional_data_handler, update_variant_acmg_classification_handler from seqr.views.utils.orm_to_json_utils import get_json_for_saved_variants -from seqr.views.utils.test_utils import AuthenticationTestCase, SAVED_VARIANT_DETAIL_FIELDS, TAG_FIELDS, GENE_VARIANT_FIELDS, \ +from seqr.views.utils.test_utils import SAVED_VARIANT_DETAIL_FIELDS, TAG_FIELDS, GENE_VARIANT_FIELDS, \ TAG_TYPE_FIELDS, LOCUS_LIST_FIELDS, PA_LOCUS_LIST_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \ - FAMILY_NOTE_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, SAVED_VARIANT_FIELDS, AnvilAuthenticationTestCase + FAMILY_NOTE_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, SAVED_VARIANT_FIELDS PROJECT_GUID = 'R0001_1kg' @@ -144,10 +144,13 @@ } -class SavedVariantAPITest(object): +class SavedVariantAPITest(ClickhouseSearchTestCase): + fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'clickhouse_discovery_variants', 'clickhouse_saved_variants'] @mock.patch('seqr.views.utils.variant_utils.OMIM_GENOME_VERSION', '37') def test_saved_variant_data(self): + Project.objects.filter(guid__in=[PROJECT_GUID, 'R0003_test']).update(genome_version='37') + url = reverse(saved_variant_data, args=[PROJECT_GUID]) self.check_collaborator_login(url) @@ -378,6 +381,17 @@ def test_saved_variant_data(self): *variant_fields, *SAVED_VARIANT_DETAIL_FIELDS, 'discoveryTags', 'noAccessDiscoveryFamilies', 'screenRegionType', 'sortedRegulatoryFeatureConsequences', 'sortedMotifFeatureConsequences', }) + self.mock_list_workspaces.assert_called_with(self.analyst_user) + self.mock_get_ws_access_level.assert_any_call( + mock.ANY, 'ext-data', 'empty') + self.mock_get_ws_access_level.assert_called_with( + mock.ANY, 'my-seqr-billing', 'anvil-1kg project n\u00e5me with uni\u00e7\u00f8de') + self.assertEqual(self.mock_get_ws_access_level.call_count, 15) + self.mock_get_groups.assert_called_with(self.collaborator_user) + self.assertEqual(self.mock_get_groups.call_count, 1) + self.mock_get_ws_acl.assert_not_called() + self.mock_get_group_members.assert_not_called() + def test_create_saved_variant(self): create_saved_variant_url = reverse(create_saved_variant_handler) self.check_collaborator_login(create_saved_variant_url, request_data={'familyGuid': 'F000001_1'}) @@ -430,6 +444,8 @@ def test_create_saved_variant(self): self.assertEqual(response.status_code, 200) self.assertListEqual(list(response.json()['savedVariantsByGuid'].keys()), [variant_guid]) + self.assert_no_list_ws_has_al(3) + def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, dataset_type='SNV_INDEL', sv_type=None, main_transcript=None): for field in ['xpos', 'ref', 'alt', 'key']: self.assertEqual(variant_json.get(field), getattr(saved_variant, field, None)) @@ -511,6 +527,8 @@ def test_create_saved_sv_variant(self): self.assertDictEqual(response_json['variantTagsByGuid'], {}) self.assertDictEqual(response_json['variantFunctionalDataByGuid'], {}) + self.assert_no_list_ws_has_al(2) + def test_create_saved_compound_hets(self): create_saved_compound_hets_url = reverse(create_saved_variant_handler) self.check_collaborator_login(create_saved_compound_hets_url, request_data={'familyGuid': 'F000001_1'}) @@ -569,6 +587,8 @@ def test_create_saved_compound_hets(self): self.assertListEqual(["Review"], [vt.variant_tag_type.name for vt in VariantTag.objects.filter( saved_variants__guid__contains=new_compound_het_4_guid)]) + self.assert_no_list_ws_has_al(2) + def test_create_manual_variant(self): create_saved_variant_url = reverse(create_manual_saved_variant_handler, args=['F000001_1']) self.check_collaborator_login(create_saved_variant_url) @@ -766,6 +786,8 @@ def test_create_update_and_delete_variant_note(self): new_variant_note = VariantNote.objects.filter(guid=updated_note_response['noteGuid']) self.assertEqual(len(new_variant_note), 0) + self.assert_no_list_ws_has_al(8) + def test_create_partially_saved_compound_het_variant_note(self): # compound het 5 is not saved, whereas compound het 1 is saved create_saved_variant_url = reverse(create_saved_variant_handler) @@ -812,6 +834,8 @@ def test_create_partially_saved_compound_het_variant_note(self): 'one_saved_one_not_saved_compount_hets_note', response_json['variantNotesByGuid'][note_guids[0]]['note'], ) + self.assert_no_list_ws_has_al(2) + def test_create_update_and_delete_compound_hets_variant_note(self): # send valid request to create variant_note for compound hets create_comp_hets_variant_note_url = reverse(create_variant_note_handler, args=[','.join([COMPOUND_HET_1_GUID, COMPOUND_HET_2_GUID])]) @@ -916,6 +940,8 @@ def test_create_update_and_delete_compound_hets_variant_note(self): variants = SavedVariant.objects.filter(guid__in=[COMPOUND_HET_1_GUID, COMPOUND_HET_2_GUID]) self.assertEqual(len(variants), 0) + self.assert_no_list_ws_has_al(7) + def test_update_variant_tags(self): variant_tags = VariantTag.objects.filter(saved_variants__guid__contains=VARIANT_GUID) self.assertSetEqual({"Review", "Tier 1 - Novel gene and phenotype"}, {vt.variant_tag_type.name for vt in variant_tags}) @@ -972,6 +998,8 @@ def test_update_variant_tags(self): self.assertEqual(VariantTag.objects.filter(saved_variants__guid__contains=COMPOUND_HET_1_GUID).count(), 0) self.assertEqual(SavedVariant.objects.filter(guid=COMPOUND_HET_1_GUID).count(), 0) + self.assert_no_list_ws_has_al(4) + def test_update_variant_functional_data(self): variant_functional_data = VariantFunctionalData.objects.filter(saved_variants__guid__contains=VARIANT_GUID) self.assertSetEqual( @@ -1010,6 +1038,8 @@ def test_update_variant_functional_data(self): {vt.functional_data_tag for vt in variant_functional_data}) self.assertSetEqual({"An updated note", "0.05"}, {vt.metadata for vt in variant_functional_data}) + self.assert_no_list_ws_has_al(2) + def test_update_compound_hets_variant_tags(self): variant_tags = VariantTag.objects.filter(saved_variants__guid__in=[COMPOUND_HET_1_GUID, COMPOUND_HET_2_GUID]) self.assertEqual(len(variant_tags), 0) @@ -1050,6 +1080,8 @@ def test_update_compound_hets_variant_tags(self): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': 'Unable to find the following variant(s): not_variant'}) + self.assert_no_list_ws_has_al(3) + def test_update_compound_hets_variant_functional_data(self): variant_functional_data = VariantFunctionalData.objects.filter( saved_variants__guid__in=[COMPOUND_HET_1_GUID, COMPOUND_HET_2_GUID]) @@ -1095,6 +1127,8 @@ def test_update_compound_hets_variant_functional_data(self): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': 'Unable to find the following variant(s): not_variant'}) + self.assert_no_list_ws_has_al(3) + def test_update_variant_main_transcript(self): transcript_id = 'ENST00000438943' update_main_transcript_url = reverse(update_variant_main_transcript, args=[VARIANT_GUID, transcript_id]) @@ -1111,6 +1145,8 @@ def test_update_variant_main_transcript(self): self.assertDictEqual(saved_variants.first().main_transcript, transcript) self.assertEqual(get_json_for_saved_variants(saved_variants)[0]['selectedMainTranscriptId'], transcript_id) + self.assert_no_list_ws_has_al(2) + def test_update_variant_acmg_classification(self): update_variant_acmg_classification_url = reverse(update_variant_acmg_classification_handler, args=[VARIANT_GUID]) self.check_collaborator_login(update_variant_acmg_classification_url) @@ -1129,87 +1165,11 @@ def test_update_variant_acmg_classification(self): self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), {'savedVariantsByGuid': {VARIANT_GUID: {'acmgClassification': variant['variant']['acmgClassification']}}}) + self.assert_no_list_ws_has_al(2) -# Tests for AnVIL access disabled -class LocalSavedVariantAPITest(AuthenticationTestCase, SavedVariantAPITest): - fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_discovery_variants', 'clickhouse_saved_variants'] - - -def assert_no_list_ws_has_al(self, acl_call_count): - self.mock_list_workspaces.assert_not_called() - self.mock_get_ws_access_level.assert_called_with(mock.ANY, - 'my-seqr-billing', 'anvil-1kg project n\u00e5me with uni\u00e7\u00f8de') - self.assertEqual(self.mock_get_ws_access_level.call_count, acl_call_count) - self.assert_no_extra_anvil_calls() - - -# Test for permissions from AnVIL only -class AnvilSavedVariantAPITest(AnvilAuthenticationTestCase, SavedVariantAPITest): - fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'clickhouse_discovery_variants', 'clickhouse_saved_variants'] - - @classmethod - def setUpTestData(cls): - super().setUpTestData() - DbnsfpGRCh37SnvIndelMv.refresh() - DbnsfpGRCh37SnvIndelDict.reload() - - def test_saved_variant_data(self, *args): - super(AnvilSavedVariantAPITest, self).test_saved_variant_data(*args) - self.mock_list_workspaces.assert_called_with(self.analyst_user) - self.mock_get_ws_access_level.assert_any_call( - mock.ANY, 'ext-data', 'empty') - self.mock_get_ws_access_level.assert_called_with( - mock.ANY, 'my-seqr-billing', 'anvil-1kg project n\u00e5me with uni\u00e7\u00f8de') - self.assertEqual(self.mock_get_ws_access_level.call_count, 15) - self.mock_get_groups.assert_called_with(self.collaborator_user) - self.assertEqual(self.mock_get_groups.call_count, 1) - self.mock_get_ws_acl.assert_not_called() - self.mock_get_group_members.assert_not_called() - - def test_create_saved_variant(self): - super(AnvilSavedVariantAPITest, self).test_create_saved_variant() - assert_no_list_ws_has_al(self, 3) - - def test_create_saved_sv_variant(self): - super(AnvilSavedVariantAPITest, self).test_create_saved_sv_variant() - assert_no_list_ws_has_al(self, 2) - - def test_create_saved_compound_hets(self): - super(AnvilSavedVariantAPITest, self).test_create_saved_compound_hets() - assert_no_list_ws_has_al(self, 2) - - def test_create_update_and_delete_variant_note(self): - super(AnvilSavedVariantAPITest, self).test_create_update_and_delete_variant_note() - assert_no_list_ws_has_al(self, 8) - - def test_create_partially_saved_compound_het_variant_note(self): - super(AnvilSavedVariantAPITest, self).test_create_partially_saved_compound_het_variant_note() - assert_no_list_ws_has_al(self, 2) - - def test_create_update_and_delete_compound_hets_variant_note(self): - super(AnvilSavedVariantAPITest, self).test_create_update_and_delete_compound_hets_variant_note() - assert_no_list_ws_has_al(self, 7) - - def test_update_variant_tags(self): - super(AnvilSavedVariantAPITest, self).test_update_variant_tags() - assert_no_list_ws_has_al(self, 4) - - def test_update_variant_functional_data(self): - super(AnvilSavedVariantAPITest, self).test_update_variant_functional_data() - assert_no_list_ws_has_al(self, 2) - - def test_update_compound_hets_variant_tags(self): - super(AnvilSavedVariantAPITest, self).test_update_compound_hets_variant_tags() - assert_no_list_ws_has_al(self, 3) - - def test_update_compound_hets_variant_functional_data(self): - super(AnvilSavedVariantAPITest, self).test_update_compound_hets_variant_functional_data() - assert_no_list_ws_has_al(self, 3) - - def test_update_variant_main_transcript(self): - super(AnvilSavedVariantAPITest, self).test_update_variant_main_transcript() - assert_no_list_ws_has_al(self, 2) - - def test_update_variant_acmg_classification(self): - super(AnvilSavedVariantAPITest, self).test_update_variant_acmg_classification() - assert_no_list_ws_has_al(self, 2) + def assert_no_list_ws_has_al(self, acl_call_count): + self.mock_list_workspaces.assert_not_called() + self.mock_get_ws_access_level.assert_called_with(mock.ANY, + 'my-seqr-billing', 'anvil-1kg project n\u00e5me with uni\u00e7\u00f8de') + self.assertEqual(self.mock_get_ws_access_level.call_count, acl_call_count) + self.assert_no_extra_anvil_calls() From 74d5a58c54217965c5022e0baf2facd2c1b2d6c3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 4 Jun 2026 12:31:45 -0400 Subject: [PATCH 162/173] test family updte metadata --- seqr/views/apis/family_api_tests.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index 6a52e6eaec..0592396816 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -456,12 +456,19 @@ def test_update_success_story_types(self): data=json.dumps({'successStoryTypes': ['O', 'D']})) self.assertEqual(response.status_code, 403) + self.reset_logs() self.login_analyst_user() response = self.client.post(url, content_type='application/json', data=json.dumps({'successStoryTypes': ['O', 'D']})) self.assertEqual(response.status_code, 200) response_json = response.json() self.assertListEqual(response_json[FAMILY_GUID]['successStoryTypes'], ['O', 'D']) + self.assert_json_logs(self.analyst_user, [ + ('update Family F000001_1', {'dbUpdate': { + 'dbEntity': 'Family', 'entityId': 'F000001_1', 'updateType': 'update', 'updateFields': ['success_story_types'], + }}), + (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}), + ]) self.check_no_analyst_no_access(url, get_response=lambda: self.client.post( url, content_type='application/json', data=json.dumps({'successStoryTypes': []}))) @@ -471,6 +478,7 @@ def test_update_family_fields(self): url = reverse(update_family_fields_handler, args=[FAMILY_GUID]) self.check_collaborator_login(url) + self.reset_logs() body = {FAMILY_ID_FIELD: 'new_id', 'description': 'Updated description', 'analysis_status': 'C'} response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 200) @@ -481,6 +489,14 @@ def test_update_family_fields(self): self.assertEqual(response_json[FAMILY_GUID]['analysisStatus'], 'C') self.assertEqual(response_json[FAMILY_GUID]['analysisStatusLastModifiedBy'], 'Test Collaborator User') self.assertEqual(response_json[FAMILY_GUID]['analysisStatusLastModifiedDate'], '2020-01-01T00:00:00') + self.assert_json_logs(self.collaborator_user, [ + ('update Family F000001_1', {'dbUpdate': { + 'dbEntity': 'Family', 'entityId': 'F000001_1', 'updateType': 'update', + 'updateFields': ['analysis_status', 'analysis_status_last_modified_by', 'analysis_status_last_modified_date', 'description'], + }}), + ('Reloading dictionary seqrdb_individual_metadata_dict', None), + (None, {'httpRequest': mock.ANY, 'requestBody': body}), + ]) # Do not update audit fields if value does not change self.login_manager() From 5034717364dfc7a7b3dee86cddd479bdaf77a2e5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 4 Jun 2026 13:06:18 -0400 Subject: [PATCH 163/173] individual tests --- seqr/views/apis/individual_api_tests.py | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index a68a391116..6c45937615 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -150,6 +150,7 @@ def test_update_individual_hpo_terms(self): edit_individuals_url = reverse(update_individual_hpo_terms, args=[INDIVIDUAL_UPDATE_GUID]) self.check_manager_login(edit_individuals_url) + self.reset_logs() response = self.client.post(edit_individuals_url, content_type='application/json', data=json.dumps(INDIVIDUAL_UPDATE_DATA)) @@ -176,6 +177,11 @@ def test_update_individual_hpo_terms(self): {'id': 'HP:0011675', 'notes': 'A new term'}, ]) + self.assert_json_logs(self.manager_user, [ + ('update Individual I000007_na20870', {'dbUpdate': mock.ANY}), + ('Reloading dictionary seqrdb_individual_metadata_dict', None), + ]) + @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP') @responses.activate def test_edit_individuals(self, mock_pm_group): @@ -378,6 +384,7 @@ def test_delete_individuals(self, mock_pm_group): dataset.active_individuals.set(set()) # send valid requests + self.reset_logs() response = self.client.post(individuals_url, content_type='application/json', data=json.dumps({ 'individuals': [INDIVIDUAL_IDS_UPDATE_DATA] })) @@ -392,6 +399,14 @@ def test_delete_individuals(self, mock_pm_group): self.assertFalse('I000002_na19678' in response_json['familiesByGuid']['F000001_1']['individualGuids']) self.assertIsNone(response_json['familiesByGuid']['F000001_1']['pedigreeImage']) self.assertFalse(Dataset.objects.filter(guid='S000130_na19678').exists()) + self.assert_json_logs(self.manager_user, [ + ('delete Dataset S000130_na19678', {'dbUpdate': mock.ANY}), + ('delete 1 Individuals', {'dbUpdate': mock.ANY}), + ('update 1 Familys', {'dbUpdate': mock.ANY}), + ('Reloading dictionary seqrdb_affected_status_dict', None), + ('Reloading dictionary seqrdb_sex_dict', None), + ('Reloading dictionary seqrdb_individual_metadata_dict', None), + ]) # Test PM permission pm_required_delete_individuals_url = reverse(delete_individuals_handler, args=[PM_REQUIRED_PROJECT_GUID]) @@ -577,6 +592,7 @@ def test_individuals_table_handler(self): url = reverse(save_individuals_table_handler, args=[PROJECT_GUID, response_json['uploadedFileId']]) + self.reset_logs() response = self.client.post(url) self.assertEqual(response.status_code, 200) response_json = response.json() @@ -611,6 +627,33 @@ def test_individuals_table_handler(self): self.assertEqual(response_json['individualsByGuid']['I000008_na20872']['individualId'], 'NA20872_update') self._assert_expected_reload_calls(PROJECT_GUID) + self.assert_json_logs(self.manager_user, [ + (mock.ANY, {'dbUpdate': { + 'dbEntity': 'Family', 'entityId': mock.ANY, 'updateFields': ['family_id', 'project'], 'updateType': 'create', + }}), + ('update Individual I000001_na19675', {'dbUpdate': mock.ANY}), + ('update Individual I000002_na19678', {'dbUpdate': mock.ANY}), + ('update Individual I000008_na20872', {'dbUpdate': mock.ANY}), + (mock.ANY, {'dbUpdate': { + 'dbEntity': 'Individual', 'entityId': mock.ANY, 'updateType': 'create', 'updateFields': [ + 'affected', 'case_review_status', 'family', 'individual_id', + ], + }}), + (mock.ANY, {'dbUpdate': { + 'dbEntity': 'FamilyNote', 'entityId': mock.ANY, 'updateType': 'create', 'updateFields': [ + 'family', 'note', 'note_type', + ], + }}), + (mock.ANY, {'dbUpdate': { + 'dbEntity': 'Individual', 'entityId': mock.ANY, 'updateType': 'update', 'updateFields': ['sex'], + }}), + ('update 3 Familys', {'dbUpdate': mock.ANY}), + ('Reloading dictionary seqrdb_affected_status_dict', None), + ('Triggering rebuild_gt_stats for R0001_1kg', None), + ('Triggered Rebuild Gt Stats', {'detail': {'project_guids': ['R0001_1kg']}}), + ('Reloading dictionary seqrdb_sex_dict', None), + ('Reloading dictionary seqrdb_individual_metadata_dict', None), + ]) # Test PM permission receive_url = reverse(receive_individuals_table_handler, args=[PM_REQUIRED_PROJECT_GUID]) @@ -1044,6 +1087,7 @@ def test_individuals_metadata_table_handler(self): ]}) # send valid request + self.reset_logs() header = f'family_id,{header}' rows[0] = '1,NA19678,,,,,false,,,,,' rows[1] = f'1,{rows[1]}' @@ -1052,6 +1096,12 @@ def test_individuals_metadata_table_handler(self): f = SimpleUploadedFile('updates.csv', "{}\n{}".format(header, '\n'.join(rows)).encode('utf-8')) response = self.client.post(url, data={'f': f}) self._is_expected_individuals_metadata_upload(response, expected_families=True, has_non_hpo_update=True) + self.assert_json_logs(self.collaborator_user, [ + (None, {'httpRequest': mock.ANY}), + ('update Individual I000002_na19678', {'dbUpdate': mock.ANY}), + ('update Individual I000001_na19675', {'dbUpdate': mock.ANY}), + ('Reloading dictionary seqrdb_individual_metadata_dict', None), + ]) def test_individuals_metadata_hpo_term_number_table_handler(self): url = reverse(receive_individuals_metadata_handler, args=['R0001_1kg']) From 6a452a587d12690ffe81fa2f2be54f7f75f79ba5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 4 Jun 2026 13:11:23 -0400 Subject: [PATCH 164/173] test project updates --- seqr/views/apis/project_api_tests.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index bc13166699..c914cffb70 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -204,6 +204,7 @@ def test_update_project(self): self.assertEqual(project.genome_version, '37') self.assertEqual(project.consent_code, 'H') + self.reset_logs() response = self.client.post(update_project_url, content_type='application/json', data=json.dumps( {'description': 'updated project description', 'restrictSharing': True, 'genomeVersion': '38', 'workspaceName': 'test update name'} )) @@ -218,6 +219,14 @@ def test_update_project(self): self.assertEqual(updated_project.description, 'updated project description') self.assertEqual(updated_project.genome_version, '37') self.assertEqual(updated_project.workspace_name, expected_workspace_name) + self.assert_json_logs(self.manager_user, [ + ('update Project R0001_1kg', {'dbUpdate': { + 'dbEntity': 'Project', 'entityId': 'R0001_1kg', 'updateType': 'update', + 'updateFields': ['description', 'restrict_sharing'], + }}), + ('Reloading dictionary seqrdb_individual_metadata_dict', None), + (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}), + ]) # test consent code response = self.client.post(update_project_url, content_type='application/json', data=json.dumps( @@ -225,12 +234,19 @@ def test_update_project(self): )) self.assertEqual(response.status_code, 403) self.login_pm_user() + self.reset_logs() response = self.client.post(update_project_url, content_type='application/json', data=json.dumps( {'consentCode': 'G'} )) self.assertEqual(response.status_code, 200) self.assertEqual(response.json()['projectsByGuid'][PROJECT_GUID]['consentCode'], 'G') self.assertEqual(Project.objects.get(guid=PROJECT_GUID).consent_code, 'G') + self.assert_json_logs(self.pm_user, [ + ('update Project R0001_1kg', {'dbUpdate': { + 'dbEntity': 'Project', 'entityId': 'R0001_1kg', 'updateType': 'update', 'updateFields': ['consent_code'], + }}), + (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}), + ]) @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', None) def test_create_project_no_pm(self): From 262deb89baa9356d228ea16e37a05c2fe82dc182 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 4 Jun 2026 14:06:38 -0400 Subject: [PATCH 165/173] test saved variant relaod --- seqr/views/apis/saved_variant_api.py | 4 +-- seqr/views/apis/saved_variant_api_tests.py | 30 ++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 8135c08f77..c6eb54f237 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -344,9 +344,9 @@ def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=Vari model.saved_variants.set(saved_variants) if has_discovery_update: - DiscoveryVariantDict.reload() + DiscoveryVariantDict.reload(user) if has_excluded_update: - ExcludedVariantDict.reload() + ExcludedVariantDict.reload(user) @login_and_policies_required diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 24046cab12..4ddddc48d3 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -949,6 +949,7 @@ def test_update_variant_tags(self): update_variant_tags_url = reverse(update_variant_tags_handler, args=[VARIANT_GUID]) self.check_collaborator_login(update_variant_tags_url, request_data={'familyGuid': 'F000001_1'}) + self.reset_logs() review_guid = 'VT1708633_2103343353_r0390_100' response = self.client.post(update_variant_tags_url, content_type='application/json', data=json.dumps({ 'tags': [ @@ -971,8 +972,19 @@ def test_update_variant_tags(self): self.assertSetEqual( {"Review", "Excluded"}, {vt.variant_tag_type.name for vt in VariantTag.objects.filter(saved_variants__guid__contains=VARIANT_GUID)}) + self.assert_json_logs(self.collaborator_user, [ + ('delete VariantTag VT1726961_2103343353_r0390_100', {'dbUpdate': mock.ANY}), + ('update VariantTag VT1708633_2103343353_r0390_100', {'dbUpdate': mock.ANY}), + (mock.ANY, {'dbUpdate': {'dbEntity': 'VariantTag', 'entityId': mock.ANY, 'updateType': 'create', 'updateFields': [ + 'metadata', 'search_hash', 'variant_tag_type', + ]}}), + ('Reloading dictionary seqrdb_discovery_variant_dict', None), + ('Reloading dictionary seqrdb_excluded_variant_dict', None), + (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}), + ]) # test delete all - with MME submission + self.reset_logs() response = self.client.post(update_variant_tags_url, content_type='application/json', data=json.dumps({ 'tags': [], 'familyGuid': 'F000001_1' @@ -984,6 +996,12 @@ def test_update_variant_tags(self): }) self.assertEqual(VariantTag.objects.filter(saved_variants__guid__contains=VARIANT_GUID).count(), 0) self.assertEqual(SavedVariant.objects.filter(guid=VARIANT_GUID).count(), 1) + self.assert_json_logs(self.collaborator_user, [ + ('delete VariantTag VT1708633_2103343353_r0390_100', {'dbUpdate': mock.ANY}), + (mock.ANY, {'dbUpdate': {'dbEntity': 'VariantTag', 'entityId': mock.ANY, 'updateType': 'delete'}}), + ('Reloading dictionary seqrdb_excluded_variant_dict', None), + (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}), + ]) # test delete all - no MME submission update_no_submission_variant_tags_url = reverse(update_variant_tags_handler, args=[COMPOUND_HET_1_GUID]) @@ -1048,6 +1066,7 @@ def test_update_compound_hets_variant_tags(self): update_variant_tags_handler, args=[','.join([COMPOUND_HET_1_GUID, COMPOUND_HET_2_GUID])]) self.check_collaborator_login(update_variant_tags_url, request_data={'familyGuid': 'F000001_1'}) + self.reset_logs() response = self.client.post(update_variant_tags_url, content_type='application/json', data=json.dumps({ 'tags': [{'name': 'Review'}, {'name': 'Excluded'}], 'familyGuid': 'F000001_1' @@ -1072,6 +1091,17 @@ def test_update_compound_hets_variant_tags(self): {vt.variant_tag_type.name for vt in VariantTag.objects.filter( saved_variants__guid__in=[COMPOUND_HET_1_GUID, COMPOUND_HET_2_GUID])}) + self.assert_json_logs(self.collaborator_user, [ + (mock.ANY, {'dbUpdate': {'dbEntity': 'VariantTag', 'entityId': mock.ANY, 'updateType': 'create', 'updateFields': [ + 'metadata', 'search_hash', 'variant_tag_type', + ]}}), + (mock.ANY, {'dbUpdate': {'dbEntity': 'VariantTag', 'entityId': mock.ANY, 'updateType': 'create', 'updateFields': [ + 'metadata', 'search_hash', 'variant_tag_type', + ]}}), + ('Reloading dictionary seqrdb_excluded_variant_dict', None), + (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}), + ]) + invalid_url = reverse(update_variant_tags_handler, args=['not_variant,{}'.format(COMPOUND_HET_1_GUID)]) response = self.client.post(invalid_url, content_type='application/json', data=json.dumps({ 'tags': [{'name': 'Review'}, {'name': 'Excluded'}], From 9aaa4b941dcabffd52253ef4e64e3eccbefb4073 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 4 Jun 2026 14:39:16 -0400 Subject: [PATCH 166/173] fix tests --- seqr/views/apis/individual_api_tests.py | 22 +++++++++++++++++++--- seqr/views/apis/saved_variant_api_tests.py | 2 +- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 6c45937615..910478d944 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -627,6 +627,8 @@ def test_individuals_table_handler(self): self.assertEqual(response_json['individualsByGuid']['I000008_na20872']['individualId'], 'NA20872_update') self._assert_expected_reload_calls(PROJECT_GUID) + read_tmp_table_logs = self._read_tmp_table_logs('ffa788c23360f0908f3de16dca799fe1') + self.assert_json_logs(None, read_tmp_table_logs) self.assert_json_logs(self.manager_user, [ (mock.ANY, {'dbUpdate': { 'dbEntity': 'Family', 'entityId': mock.ANY, 'updateFields': ['family_id', 'project'], 'updateType': 'create', @@ -653,7 +655,7 @@ def test_individuals_table_handler(self): ('Triggered Rebuild Gt Stats', {'detail': {'project_guids': ['R0001_1kg']}}), ('Reloading dictionary seqrdb_sex_dict', None), ('Reloading dictionary seqrdb_individual_metadata_dict', None), - ]) + ], offset=len(read_tmp_table_logs)) # Test PM permission receive_url = reverse(receive_individuals_table_handler, args=[PM_REQUIRED_PROJECT_GUID]) @@ -676,6 +678,10 @@ def _assert_expected_reload_calls(self, project_guid): self.assertEqual(len(responses.calls), 1) self.assertDictEqual(json.loads(responses.calls[0].request.body), {'project_guids': [project_guid]}) + @staticmethod + def _read_tmp_table_logs(file_name): + return [] + @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') @mock.patch('seqr.views.utils.pedigree_info_utils.NO_VALIDATE_MANIFEST_PROJECT_CATEGORIES') @mock.patch('seqr.utils.communication_utils.EmailMultiAlternatives') @@ -1096,12 +1102,15 @@ def test_individuals_metadata_table_handler(self): f = SimpleUploadedFile('updates.csv', "{}\n{}".format(header, '\n'.join(rows)).encode('utf-8')) response = self.client.post(url, data={'f': f}) self._is_expected_individuals_metadata_upload(response, expected_families=True, has_non_hpo_update=True) + + read_tmp_table_logs = self._read_tmp_table_logs('6aea3d1f1bfc295340aa504370102fd5') + offset = 2 if read_tmp_table_logs else 1 + self.assert_json_logs(None, read_tmp_table_logs, offset=offset) self.assert_json_logs(self.collaborator_user, [ - (None, {'httpRequest': mock.ANY}), ('update Individual I000002_na19678', {'dbUpdate': mock.ANY}), ('update Individual I000001_na19675', {'dbUpdate': mock.ANY}), ('Reloading dictionary seqrdb_individual_metadata_dict', None), - ]) + ], offset=offset+len(read_tmp_table_logs)) def test_individuals_metadata_hpo_term_number_table_handler(self): url = reverse(receive_individuals_metadata_handler, args=['R0001_1kg']) @@ -1518,3 +1527,10 @@ def _mock_subprocess(self, command, **kwargs): else: self.mock_subprocess.stdout.__iter__.return_value = self.gs_files[file_name] return self.mock_subprocess + + @staticmethod + def _read_tmp_table_logs(file_name): + return [ + (f'==> gsutil ls gs://seqr-scratch-temp/temp_upload_{file_name}.json.gz', None), + (f'==> gsutil cat gs://seqr-scratch-temp/temp_upload_{file_name}.json.gz | gunzip -c -q - ', None), + ] diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 4ddddc48d3..c13cca28f2 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -997,7 +997,7 @@ def test_update_variant_tags(self): self.assertEqual(VariantTag.objects.filter(saved_variants__guid__contains=VARIANT_GUID).count(), 0) self.assertEqual(SavedVariant.objects.filter(guid=VARIANT_GUID).count(), 1) self.assert_json_logs(self.collaborator_user, [ - ('delete VariantTag VT1708633_2103343353_r0390_100', {'dbUpdate': mock.ANY}), + (mock.ANY, {'dbUpdate': {'dbEntity': 'VariantTag', 'entityId': mock.ANY, 'updateType': 'delete'}}), (mock.ANY, {'dbUpdate': {'dbEntity': 'VariantTag', 'entityId': mock.ANY, 'updateType': 'delete'}}), ('Reloading dictionary seqrdb_excluded_variant_dict', None), (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}), From 0bafd439d2904b79c21425654c0f8def8fa6ffc9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 4 Jun 2026 15:51:22 -0400 Subject: [PATCH 167/173] pr feedback --- seqr/views/apis/family_api.py | 4 +++- seqr/views/apis/saved_variant_api.py | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index dc4bde189c..493bc8a380 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -5,7 +5,7 @@ from django.db.models import Count, Max, Q, F from django.db.models.fields.files import ImageFieldFile -from clickhouse_search.models.postgres_dicts import IndividualMetadataDict +from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict, IndividualMetadataDict from matchmaker.models import MatchmakerSubmission from reference_data.models import Omim from seqr.utils.gene_utils import get_genes_for_variant_display @@ -211,6 +211,8 @@ def delete_families_handler(request, project_guid): # delete families Family.bulk_delete(request.user, project=project, guid__in=family_guids_to_delete) + AffectedDict.reload(request.user) + SexDict.reload(request.user) IndividualMetadataDict.reload(request.user) # send response diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index c6eb54f237..68f99caf28 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -313,11 +313,16 @@ def _delete_removed_tags(saved_variants, all_variant_guids, tag_updates, user, t if protected_tag_types: remove_tags = remove_tags.exclude(variant_tag_type__name__in=protected_tag_types) track_updates = tag_type == 'tag' - has_discovery_update = remove_tags.filter(variant_tag_type__category=DISCOVERY_CATEGORY).exists() if track_updates else False - has_excluded_update = remove_tags.filter(variant_tag_type__name='Excluded').exists() if track_updates else False + has_discovery_update = False + has_excluded_update = False for tag in remove_tags: tag_variant_guids = {sv.guid for sv in tag.saved_variants.all()} if tag_variant_guids == all_variant_guids: + if track_updates: + if tag.variant_tag_type.category == DISCOVERY_CATEGORY: + has_discovery_update = True + elif tag.variant_tag_type.name == 'Excluded': + has_excluded_update = True deleted_tag_guids.append(tag.guid) tag.delete_model(user, user_can_delete=True) return deleted_tag_guids, has_discovery_update, has_excluded_update From 692e3ff19775a39a4611379f104e735f63c96559 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 8 Jun 2026 12:44:29 -0400 Subject: [PATCH 168/173] actually merge dev --- .github/workflows/unit-tests.yml | 26 +++++++++++++++++++++----- .github/workflows/vlm-unit-tests.yaml | 3 ++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index daf1dfe876..fae9c684e8 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -41,9 +41,10 @@ jobs: # Maps tcp port 5432 on service container to the host - 5432:5432 clickhouse: - image: bitnamilegacy/clickhouse:latest + image: gcr.io/seqr-project/seqr-clickhouse:26.3.9 ports: - 9000:9000 # Native client interface + - 8123:8123 # HTTP interface volumes: - /var/tmp:/var/seqr/clickhouse-data - /tmp:/in-memory-dir @@ -57,12 +58,13 @@ jobs: CLICKHOUSE_USER: clickhouse_test CLICKHOUSE_PASSWORD: clickhouse_test ALLOW_EMPTY_PASSWORD: no + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 steps: - uses: actions/checkout@v2 - name: Set up Clickhouse Settings - run: | - cat < test_config.xml + run: | + cat < test_user_config.xml @@ -71,13 +73,20 @@ jobs: EOF - docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml + docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'" - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1 TO clickhouse_test, clickhouse_read_only" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, enable_join_runtime_filters=0 TO clickhouse_test, clickhouse_read_only" docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" + cat < test_config.xml + + / + + EOF + docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml + docker restart clickhouse - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: @@ -94,6 +103,13 @@ jobs: python -m pip install --upgrade pip wheel pip install -r requirements.txt pip install -r requirements-dev.txt + - name: Wait for service restart + run: | + timeout 50s sh -c ' + until curl --fail http://localhost:8123/ping; do + sleep 5 + done + ' - name: Run coverage tests run: | export CLICKHOUSE_READER_USER=clickhouse_read_only diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index c8cf522db2..cf2f50c7fe 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -26,7 +26,7 @@ jobs: services: clickhouse: - image: bitnamilegacy/clickhouse:latest + image: gcr.io/seqr-project/seqr-clickhouse:26.3.9 ports: - 8123:8123 # HTTP interface - 9000:9000 # Native client interface @@ -43,6 +43,7 @@ jobs: CLICKHOUSE_USER: clickhouse_test_user CLICKHOUSE_PASSWORD: clickhouse_test_password ALLOW_EMPTY_PASSWORD: no + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 postgres: image: postgres ports: From 5738357b40529048f52647db0909909624f5cc8a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 8 Jun 2026 12:53:37 -0400 Subject: [PATCH 169/173] update for newwer verrsion --- .github/workflows/vlm-unit-tests.yaml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index cf2f50c7fe..5c98755886 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -70,7 +70,7 @@ jobs: pip install -r vlm/requirements-test.txt - name: Set up Clickhouse Settings run: | - cat < test_config.xml + cat < test_user_config.xml @@ -79,12 +79,26 @@ jobs: EOF - docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml + docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1 TO vlm_test_user, clickhouse_test_user" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" + cat < test_config.xml + + / + + EOF + docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml + docker restart clickhouse + - name: Wait for service restart + run: | + timeout 50s sh -c ' + until curl --fail http://localhost:8123/ping; do + sleep 5 + done + ' - name: Set up Clickhouse Test Database run: | python3 -m venv .venv From 0d7c2ccb49d13ce7b1a472ff43b31d9d03b3cc2e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 8 Jun 2026 13:40:20 -0400 Subject: [PATCH 170/173] clean up db config for clickhouse --- settings.py | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/settings.py b/settings.py index 5c3871d8cf..532984712a 100644 --- a/settings.py +++ b/settings.py @@ -234,36 +234,35 @@ 'USER': os.environ.get('POSTGRES_USERNAME', 'postgres'), 'PASSWORD': os.environ.get('POSTGRES_PASSWORD', 'pgtest'), } +CLICKHOUSE_DB_CONFIG = { + 'ENGINE': 'clickhouse_search.backend', + 'NAME': 'seqr', + 'HOST': os.environ.get('CLICKHOUSE_SERVICE_HOSTNAME', 'localhost'), + 'PORT': int(os.environ.get('CLICKHOUSE_SERVICE_PORT', '9000')), + 'OPTIONS': { + 'settings': { + 'use_client_time_zone': False, + } + } +} DATABASES = { 'default': dict(NAME='seqrdb', **POSTGRES_DB_CONFIG), 'reference_data': dict(NAME='reference_data_db', **POSTGRES_DB_CONFIG), + 'clickhouse_write': { + **CLICKHOUSE_DB_CONFIG, + 'USER': os.environ.get('CLICKHOUSE_WRITER_USER', 'clickhouse'), + 'PASSWORD': os.environ.get('CLICKHOUSE_WRITER_PASSWORD', 'clickhouse_test'), + }, + 'clickhouse': { + **CLICKHOUSE_DB_CONFIG, + 'USER': os.environ.get('CLICKHOUSE_READER_USER', 'clickhouse'), + 'PASSWORD': os.environ.get('CLICKHOUSE_READER_PASSWORD', 'clickhouse_test'), + }, } DATABASE_ROUTERS = ['reference_data.models.ReferenceDataRouter', 'clickhouse_search.models.ClickHouseRouter'] CLICKHOUSE_IN_MEMORY_DIR = os.environ.get('CLICKHOUSE_IN_MEMORY_DIR', '/in-memory-dir') CLICKHOUSE_DATA_DIR = os.getenv('CLICKHOUSE_DATA_DIR', '/var/seqr/clickhouse-data') -CLICKHOUSE_SERVICE_HOSTNAME = os.environ.get('CLICKHOUSE_SERVICE_HOSTNAME') -CLICKHOUSE_WRITER_USER = os.environ.get('CLICKHOUSE_WRITER_USER', 'clickhouse') -CLICKHOUSE_WRITER_PASSWORD = os.environ.get('CLICKHOUSE_WRITER_PASSWORD', 'clickhouse_test') -if CLICKHOUSE_SERVICE_HOSTNAME: - DATABASES['clickhouse_write'] = { - 'ENGINE': 'clickhouse_search.backend', - 'NAME': 'seqr', - 'HOST': CLICKHOUSE_SERVICE_HOSTNAME, - 'PORT': int(os.environ.get('CLICKHOUSE_SERVICE_PORT', '9000')), - 'USER': CLICKHOUSE_WRITER_USER, - 'PASSWORD': CLICKHOUSE_WRITER_PASSWORD, - 'OPTIONS': { - 'settings': { - 'use_client_time_zone': False, - } - }, - } - DATABASES['clickhouse'] = { - **DATABASES['clickhouse_write'], - 'USER': os.environ.get('CLICKHOUSE_READER_USER', 'clickhouse'), - 'PASSWORD': os.environ.get('CLICKHOUSE_READER_PASSWORD', 'clickhouse_test'), - } TEST_RUNNER = "seqr.testrunner.OrderedDatabaseDeletionRunner" From 5aef7070da944fecb8a91db2d69f5fbbd8e159d0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 8 Jun 2026 13:48:15 -0400 Subject: [PATCH 171/173] fix settings vars --- settings.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/settings.py b/settings.py index 532984712a..7c21b1584e 100644 --- a/settings.py +++ b/settings.py @@ -245,13 +245,15 @@ } } } +CLICKHOUSE_WRITER_USER = os.environ.get('CLICKHOUSE_WRITER_USER', 'clickhouse') +CLICKHOUSE_WRITER_PASSWORD = os.environ.get('CLICKHOUSE_WRITER_PASSWORD', 'clickhouse_test') DATABASES = { 'default': dict(NAME='seqrdb', **POSTGRES_DB_CONFIG), 'reference_data': dict(NAME='reference_data_db', **POSTGRES_DB_CONFIG), 'clickhouse_write': { **CLICKHOUSE_DB_CONFIG, - 'USER': os.environ.get('CLICKHOUSE_WRITER_USER', 'clickhouse'), - 'PASSWORD': os.environ.get('CLICKHOUSE_WRITER_PASSWORD', 'clickhouse_test'), + 'USER': CLICKHOUSE_WRITER_USER, + 'PASSWORD': CLICKHOUSE_WRITER_PASSWORD, }, 'clickhouse': { **CLICKHOUSE_DB_CONFIG, From 3e44ff6e8b4a3d930369a4d0a0cc60d9bd2a61d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jun 2026 14:28:44 +0000 Subject: [PATCH 172/173] Bump shell-quote from 1.8.1 to 1.8.4 in /ui Bumps [shell-quote](https://github.com/ljharb/shell-quote) from 1.8.1 to 1.8.4. - [Changelog](https://github.com/ljharb/shell-quote/blob/main/CHANGELOG.md) - [Commits](https://github.com/ljharb/shell-quote/compare/v1.8.1...v1.8.4) --- updated-dependencies: - dependency-name: shell-quote dependency-version: 1.8.4 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- ui/package-lock.json | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index 94c0415ba7..cbbc6ace05 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -16113,10 +16113,13 @@ } }, "node_modules/shell-quote": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.1.tgz", - "integrity": "sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==", + "version": "1.8.4", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.4.tgz", + "integrity": "sha512-VsC6n6vz1ihYYyZZwX7YZSF5l5x36ca17OC+a69h94YqB7X6XLwf+5MOgynYir2SLFUbl8gIYvBo8K8RoNQ6bQ==", "dev": true, + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -31977,9 +31980,9 @@ "dev": true }, "shell-quote": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.1.tgz", - "integrity": "sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==", + "version": "1.8.4", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.4.tgz", + "integrity": "sha512-VsC6n6vz1ihYYyZZwX7YZSF5l5x36ca17OC+a69h94YqB7X6XLwf+5MOgynYir2SLFUbl8gIYvBo8K8RoNQ6bQ==", "dev": true }, "side-channel": { From 71c12eff2e8fb4eb1c67c82e0d2e25acb93e77b3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 10 Jun 2026 13:05:16 -0400 Subject: [PATCH 173/173] correctly mock response in tests --- seqr/views/apis/report_api_tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 2edbc90742..3e46cf9680 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -1331,10 +1331,18 @@ def test_family_metadata(self): self.assertListEqual( sorted([r['familyGuid'] for r in response.json()['rows']]), expected_families + self.ADDITIONAL_FAMILIES) + @responses.activate def test_variant_metadata(self): url = reverse(variant_metadata, args=[PROJECT_GUID]) self.check_analyst_login(url) + responses.add(responses.GET, 'https://monarchinitiative.org/v3/api/entity/MONDO:0044970', status=200, json={ + 'id': 'MONDO:0044970', + 'category': 'biolink:Disease', + 'name': 'mitochondrial disease', + 'inheritance': {}, + }) + response = self.client.get(url) self.assertEqual(response.status_code, 200) response_json = response.json()