Dev

.github/workflows/unit-tests.yml

@@ -41,9 +41,10 @@ jobs:
           # Maps tcp port 5432 on service container to the host
           - 5432:5432
       clickhouse:
-        image: bitnamilegacy/clickhouse:latest
+        image: gcr.io/seqr-project/seqr-clickhouse:26.3.9
         ports:
           - 9000:9000   # Native client interface
+          - 8123:8123   # HTTP interface
         volumes:
           - /var/tmp:/var/seqr/clickhouse-data
           - /tmp:/in-memory-dir
@@ -57,12 +58,13 @@ jobs:
           CLICKHOUSE_USER: clickhouse_test
           CLICKHOUSE_PASSWORD: clickhouse_test
           ALLOW_EMPTY_PASSWORD: no
+          CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
 
     steps:
       - uses: actions/checkout@v2
       - name: Set up Clickhouse Settings
-        run: |
-          cat <<EOF > test_config.xml
+        run: |          
+          cat <<EOF > test_user_config.xml
           <clickhouse>
             <users>
               <clickhouse_test>
@@ -71,13 +73,20 @@ jobs:
             </users>
           </clickhouse>
           EOF
-          docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml
+          docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml
           docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS"
           docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'"
-          docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1 TO clickhouse_test, clickhouse_read_only"
+          docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, enable_join_runtime_filters=0 TO clickhouse_test, clickhouse_read_only"
           docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only"
           docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'"
           docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS"
+          cat <<EOF > test_config.xml
+          <clickhouse>
+            <user_files_path>/</user_files_path>
+          </clickhouse>
+          EOF
+          docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml
+          docker restart clickhouse
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
@@ -94,6 +103,13 @@ jobs:
           python -m pip install --upgrade pip wheel
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
+      - name: Wait for service restart
+        run: |
+          timeout 50s sh -c '
+          until curl --fail http://localhost:8123/ping; do
+            sleep 5
+          done
+          '
       - name: Run coverage tests
         run: |
           export CLICKHOUSE_READER_USER=clickhouse_read_only

.github/workflows/vlm-unit-tests.yaml

@@ -12,22 +12,27 @@ on:
     paths:
       - 'vlm/**'
       - '.github/workflows/*vlm*.yaml'
+      - 'clickhouse_search/fixtures/clickhouse_search.json'
   pull_request:
     types: [opened, synchronize, reopened]
     paths:
       - 'vlm/**'
       - '.github/workflows/*vlm*.yaml'
+      - 'clickhouse_search/fixtures/clickhouse_search.json'
 
 jobs:
   vlm_clickhouse:
     runs-on: ubuntu-latest
-    container: python:3.11-slim-bullseye
 
     services:
       clickhouse:
-        image: bitnamilegacy/clickhouse:latest
+        image: gcr.io/seqr-project/seqr-clickhouse:26.3.9
         ports:
           - 8123:8123   # HTTP interface
+          - 9000:9000   # Native client interface
+        volumes:
+          - /var/tmp:/var/seqr/clickhouse-data
+          - /tmp:/in-memory-dir
         options: >-
           --health-cmd "clickhouse-client --query 'SELECT 1'"
           --health-interval 10s 
@@ -38,25 +43,92 @@ jobs:
           CLICKHOUSE_USER: clickhouse_test_user
           CLICKHOUSE_PASSWORD: clickhouse_test_password
           ALLOW_EMPTY_PASSWORD: no
+          CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+      postgres:
+        image: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        env:
+          POSTGRES_PASSWORD: pgtest
 
     steps:
       - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+          cache: 'pip'
       - name: Install dependencies
         run: |
           python3 -m pip install --upgrade pip wheel
           pip install -r vlm/requirements.txt
           pip install -r vlm/requirements-test.txt
-      - name: Set up Clickhouse Settings and Data
-        run: python3 vlm/setup_clickhouse_test_data.py clickhouse 8123 clickhouse_test_user clickhouse_test_password
+      - name: Set up Clickhouse Settings
+        run: |
+          cat <<EOF > test_user_config.xml
+          <clickhouse>
+            <users>
+              <clickhouse_test_user>
+                <named_collection_control>1</named_collection_control>
+              </clickhouse_test_user>
+            </users>
+          </clickhouse>
+          EOF
+          docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml
+          docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'"
+          docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS"
+          docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1 TO vlm_test_user, clickhouse_test_user"
+          docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'"
+          docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS"
+          cat <<EOF > test_config.xml
+          <clickhouse>
+            <user_files_path>/</user_files_path>
+          </clickhouse>
+          EOF
+          docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml
+          docker restart clickhouse
+      - name: Wait for service restart
+        run: |
+          timeout 50s sh -c '
+          until curl --fail http://localhost:8123/ping; do
+            sleep 5
+          done
+          '
+      - name: Set up Clickhouse Test Database
+        run: | 
+          python3 -m venv .venv
+          source .venv/bin/activate
+          pip install -r vlm/requirements.txt
+          pip install -r vlm/requirements-test.txt
+          pip install pytest-django
+          pip install -r requirements.txt 
+          pip install -r requirements-dev.txt 
+          export CLICKHOUSE_WRITER_USER=clickhouse_test_user
+          export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password
+          export CLICKHOUSE_SERVICE_HOSTNAME=localhost
+          export POSTGRES_SERVICE_HOSTNAME=localhost
+          python3 -m pytest --setup-only --create-db --reuse-db --ds=settings
+      - name: Set up Clickhouse test user
+        run: |
+          docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.\`GRCh37/SNV_INDEL/key_lookup\` TO vlm_test_user"
+          docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.\`GRCh38/SNV_INDEL/key_lookup\` TO vlm_test_user"
+          docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.\`GRCh37/SNV_INDEL/gt_stats_dict\` TO vlm_test_user"
+          docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.\`GRCh38/SNV_INDEL/gt_stats_dict\` TO vlm_test_user"
+          docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS"
       - name: Run coverage tests
         run: |
           export SEQR_BASE_URL=https://test-seqr.org/
           export NODE_ID=TestVLM
-          export CLICKHOUSE_SERVICE_HOSTNAME=clickhouse
+          export CLICKHOUSE_SERVICE_HOSTNAME=localhost
           export CLICKHOUSE_SERVICE_PORT=8123
           export CLICKHOUSE_VLM_USERNAME=vlm_test_user
           export CLICKHOUSE_VLM_PASSWORD=vlm_test_password
           export CLICKHOUSE_DATABASE=test_seqr
-          coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py" -m pytest vlm/
+          coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/conftest.py" -m pytest vlm/
           coverage report -m --fail-under=95
 

reference_data/management/commands/update_all_reference_data.py

@@ -2,6 +2,7 @@
 from collections import OrderedDict
 from django.core.management.base import BaseCommand, CommandError
 
+from clickhouse_search.models.postgres_dicts import GeneIdDict
 from panelapp.models import PanelAppAU, PanelAppUK
 from reference_data.utils.gene_utils import get_genes_by_id_and_symbol
 from reference_data.models import GeneInfo, TranscriptInfo, HumanPhenotypeOntology, RefseqTranscript, GeneConstraint, \
@@ -57,6 +58,8 @@ def handle(self, *args, **options):
             data_model_name = GeneInfo.__name__
             self._update_gencode(current_versions.get(data_model_name), options['gene_symbol_change_dir'])
             self._track_success_updates(data_model_name, latest_version, current_versions, updated)
+            GeneIdDict.reload()
+
 
         gene_ids_to_gene, gene_symbols_to_gene = get_genes_by_id_and_symbol() if to_update else (None, None)
         for data_cls, latest_version in to_update.items():

reference_data/management/tests/update_all_reference_data_tests.py

@@ -42,6 +42,7 @@ def _mock_handler(_cls, **kwargs):
 
 
 class NewDbUpdateAllReferenceDataTest(BaseUpdateAllReferenceDataTest):
+    databases = '__all__'
     fixtures = ['users']
 
     def test_empty_db_update_all_reference_data_command(self):
@@ -78,6 +79,7 @@ def test_empty_db_update_all_reference_data_command(self):
 
         self.mock_slack.assert_not_called()
         self.assert_json_logs(user=None, expected=[
+            ('Reloading dictionary seqrdb_gene_ids', None),
             ('unable to update PrimateAI: Primate_AI failed', {
                 'severity': 'ERROR',
                 '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent',

reference_data/management/tests/update_gencode_tests.py

@@ -54,6 +54,8 @@
 
 class UpdateGencodeTest(ReferenceDataCommandTestCase):
 
+    databases = '__all__'
+
     def setUp(self):
         super().setUp()
 
@@ -202,6 +204,7 @@ def test_update_gencode_latest_command(self):
             ('Loaded 3 RefseqTranscript records', None),
             ('Skipped 1 records with unrecognized or duplicated transcripts', None),
             ('Updated GeneInfo reference data from version "31" to version "39"', None),
+            ('Reloading dictionary seqrdb_gene_ids', None),
             ('Done', None),
             ('Updated: GeneInfo', None),
         ])

seqr/views/apis/family_api.py

@@ -5,6 +5,7 @@
 from django.db.models import Count, Max, Q, F
 from django.db.models.fields.files import ImageFieldFile
 
+from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict, IndividualMetadataDict
 from matchmaker.models import MatchmakerSubmission
 from reference_data.models import Omim
 from seqr.utils.gene_utils import get_genes_for_variant_display
@@ -210,6 +211,10 @@ def delete_families_handler(request, project_guid):
     # delete families
     Family.bulk_delete(request.user, project=project, guid__in=family_guids_to_delete)
 
+    AffectedDict.reload(request.user)
+    SexDict.reload(request.user)
+    IndividualMetadataDict.reload(request.user)
+
     # send response
     return create_json_response({
         'individualsByGuid': {
@@ -230,9 +235,12 @@ def update_family_fields_handler(request, family_guid):
 
     request_json = json.loads(request.body)
     immutable_keys = [] if external_anvil_project_can_edit(family.project, request.user) else ['family_id']
+    updated_fields = set()
     update_family_from_json(family, request_json, user=request.user, allow_unknown_keys=True, immutable_keys=[
         'display_name',
-    ] + immutable_keys)
+    ] + immutable_keys, updated_fields=updated_fields)
+    if updated_fields.intersection({'post_discovery_omim_numbers', 'post_discovery_mondo_id', 'analysis_status'}):
+        IndividualMetadataDict.reload(request.user)
 
     return create_json_response({
         family.guid: _get_json_for_model(family, user=request.user, process_result=_set_display_name)

seqr/views/apis/family_api_tests.py

@@ -456,12 +456,19 @@ def test_update_success_story_types(self):
                                     data=json.dumps({'successStoryTypes': ['O', 'D']}))
         self.assertEqual(response.status_code, 403)
 
+        self.reset_logs()
         self.login_analyst_user()
         response = self.client.post(url, content_type='application/json',
                                     data=json.dumps({'successStoryTypes': ['O', 'D']}))
         self.assertEqual(response.status_code, 200)
         response_json = response.json()
         self.assertListEqual(response_json[FAMILY_GUID]['successStoryTypes'], ['O', 'D'])
+        self.assert_json_logs(self.analyst_user, [
+            ('update Family F000001_1', {'dbUpdate': {
+                'dbEntity': 'Family', 'entityId': 'F000001_1', 'updateType': 'update', 'updateFields': ['success_story_types'],
+            }}),
+            (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}),
+        ])
 
         self.check_no_analyst_no_access(url, get_response=lambda: self.client.post(
             url, content_type='application/json', data=json.dumps({'successStoryTypes': []})))
@@ -471,6 +478,7 @@ def test_update_family_fields(self):
         url = reverse(update_family_fields_handler, args=[FAMILY_GUID])
         self.check_collaborator_login(url)
 
+        self.reset_logs()
         body = {FAMILY_ID_FIELD: 'new_id', 'description': 'Updated description', 'analysis_status': 'C'}
         response = self.client.post(url, content_type='application/json', data=json.dumps(body))
         self.assertEqual(response.status_code, 200)
@@ -481,6 +489,14 @@ def test_update_family_fields(self):
         self.assertEqual(response_json[FAMILY_GUID]['analysisStatus'], 'C')
         self.assertEqual(response_json[FAMILY_GUID]['analysisStatusLastModifiedBy'], 'Test Collaborator User')
         self.assertEqual(response_json[FAMILY_GUID]['analysisStatusLastModifiedDate'], '2020-01-01T00:00:00')
+        self.assert_json_logs(self.collaborator_user, [
+            ('update Family F000001_1', {'dbUpdate': {
+                'dbEntity': 'Family', 'entityId': 'F000001_1', 'updateType': 'update',
+                'updateFields': ['analysis_status', 'analysis_status_last_modified_by', 'analysis_status_last_modified_date', 'description'],
+            }}),
+            ('Reloading dictionary seqrdb_individual_metadata_dict', None),
+            (None, {'httpRequest': mock.ANY, 'requestBody': body}),
+        ])
 
         # Do not update audit fields if value does not change
         self.login_manager()

seqr/views/apis/individual_api.py

@@ -8,6 +8,7 @@
 from django.contrib.auth.models import User
 from django.db.models import prefetch_related_objects
 
+from clickhouse_search.models.postgres_dicts import IndividualMetadataDict, SexDict, AffectedDict
 from reference_data.models import HumanPhenotypeOntology
 from seqr.models import Individual, Family, CAN_VIEW
 from seqr.utils.file_utils import file_iter
@@ -93,6 +94,7 @@ def update_individual_hpo_terms(request, individual_guid):
         for key in feature_fields
     }
     update_individual_from_json(individual, update_json, user=request.user, allow_features_update=True)
+    IndividualMetadataDict.reload(request.user)
 
     individual_json = {k: getattr(individual, _to_snake_case(k)) for k in feature_fields}
     add_individual_hpo_details([individual_json])
@@ -218,6 +220,9 @@ def delete_individuals_handler(request, project_guid):
 
     # delete the individuals
     families_with_deleted_individuals = delete_individuals(project, individual_guids_to_delete, request.user)
+    AffectedDict.reload(request.user)
+    SexDict.reload(request.user)
+    IndividualMetadataDict.reload(request.user)
 
     deleted_individuals_by_guid = {
         individual_guid: None for individual_guid in individual_guids_to_delete
@@ -665,6 +670,9 @@ def save_individuals_metadata_table_handler(request, project_guid, upload_file_i
         if record.get(ASSIGNED_ANALYST_COL):
             family_assigned_analysts[record[ASSIGNED_ANALYST_COL]].append(individual.family.id)
 
+    if any(FEATURES_COL in record for record in json_records):
+        IndividualMetadataDict.reload(request.user)
+
     response = {
         'individualsByGuid': {
             individual['individualGuid']: individual for individual in _get_json_for_individuals(