diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index daf1dfe876..fae9c684e8 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -41,9 +41,10 @@ jobs: # Maps tcp port 5432 on service container to the host - 5432:5432 clickhouse: - image: bitnamilegacy/clickhouse:latest + image: gcr.io/seqr-project/seqr-clickhouse:26.3.9 ports: - 9000:9000 # Native client interface + - 8123:8123 # HTTP interface volumes: - /var/tmp:/var/seqr/clickhouse-data - /tmp:/in-memory-dir @@ -57,12 +58,13 @@ jobs: CLICKHOUSE_USER: clickhouse_test CLICKHOUSE_PASSWORD: clickhouse_test ALLOW_EMPTY_PASSWORD: no + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 steps: - uses: actions/checkout@v2 - name: Set up Clickhouse Settings - run: | - cat < test_config.xml + run: | + cat < test_user_config.xml @@ -71,13 +73,20 @@ jobs: EOF - docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml + docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" docker exec clickhouse clickhouse-client --query "CREATE USER clickhouse_read_only IDENTIFIED WITH plaintext_password BY 'clickhouse_test'" - docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1 TO clickhouse_test, clickhouse_read_only" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, enable_join_runtime_filters=0 TO clickhouse_test, clickhouse_read_only" docker exec clickhouse clickhouse-client --query "GRANT SELECT, SYSTEM VIEWS, dictGet ON *.* TO clickhouse_read_only" docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" + cat < test_config.xml + + / + + EOF + docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml + docker restart clickhouse - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: @@ -94,6 +103,13 @@ jobs: python -m pip install --upgrade pip wheel pip install -r requirements.txt pip install -r requirements-dev.txt + - name: Wait for service restart + run: | + timeout 50s sh -c ' + until curl --fail http://localhost:8123/ping; do + sleep 5 + done + ' - name: Run coverage tests run: | export CLICKHOUSE_READER_USER=clickhouse_read_only diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index 6ac26e1a35..5c98755886 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -12,22 +12,27 @@ on: paths: - 'vlm/**' - '.github/workflows/*vlm*.yaml' + - 'clickhouse_search/fixtures/clickhouse_search.json' pull_request: types: [opened, synchronize, reopened] paths: - 'vlm/**' - '.github/workflows/*vlm*.yaml' + - 'clickhouse_search/fixtures/clickhouse_search.json' jobs: vlm_clickhouse: runs-on: ubuntu-latest - container: python:3.11-slim-bullseye services: clickhouse: - image: bitnamilegacy/clickhouse:latest + image: gcr.io/seqr-project/seqr-clickhouse:26.3.9 ports: - 8123:8123 # HTTP interface + - 9000:9000 # Native client interface + volumes: + - /var/tmp:/var/seqr/clickhouse-data + - /tmp:/in-memory-dir options: >- --health-cmd "clickhouse-client --query 'SELECT 1'" --health-interval 10s @@ -38,25 +43,92 @@ jobs: CLICKHOUSE_USER: clickhouse_test_user CLICKHOUSE_PASSWORD: clickhouse_test_password ALLOW_EMPTY_PASSWORD: no + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 + postgres: + image: postgres + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + POSTGRES_PASSWORD: pgtest steps: - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11 + cache: 'pip' - name: Install dependencies run: | python3 -m pip install --upgrade pip wheel pip install -r vlm/requirements.txt pip install -r vlm/requirements-test.txt - - name: Set up Clickhouse Settings and Data - run: python3 vlm/setup_clickhouse_test_data.py clickhouse 8123 clickhouse_test_user clickhouse_test_password + - name: Set up Clickhouse Settings + run: | + cat < test_user_config.xml + + + + 1 + + + + EOF + docker cp ${{ github.workspace }}/test_user_config.xml clickhouse:/etc/clickhouse-server/users.d/users.xml + docker exec clickhouse clickhouse-client --query "CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'" + docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" + docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1 TO vlm_test_user, clickhouse_test_user" + docker exec clickhouse clickhouse-client --query "CREATE NAMED COLLECTION seqr_postgres_named_collection AS user='postgres', password='pgtest', host='postgres', port='5432'" + docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" + cat < test_config.xml + + / + + EOF + docker cp ${{ github.workspace }}/test_config.xml clickhouse:/etc/clickhouse-server/config.d/config.xml + docker restart clickhouse + - name: Wait for service restart + run: | + timeout 50s sh -c ' + until curl --fail http://localhost:8123/ping; do + sleep 5 + done + ' + - name: Set up Clickhouse Test Database + run: | + python3 -m venv .venv + source .venv/bin/activate + pip install -r vlm/requirements.txt + pip install -r vlm/requirements-test.txt + pip install pytest-django + pip install -r requirements.txt + pip install -r requirements-dev.txt + export CLICKHOUSE_WRITER_USER=clickhouse_test_user + export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password + export CLICKHOUSE_SERVICE_HOSTNAME=localhost + export POSTGRES_SERVICE_HOSTNAME=localhost + python3 -m pytest --setup-only --create-db --reuse-db --ds=settings + - name: Set up Clickhouse test user + run: | + docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.\`GRCh37/SNV_INDEL/key_lookup\` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.\`GRCh38/SNV_INDEL/key_lookup\` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.\`GRCh37/SNV_INDEL/gt_stats_dict\` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "GRANT dictGet ON test_seqr.\`GRCh38/SNV_INDEL/gt_stats_dict\` TO vlm_test_user" + docker exec clickhouse clickhouse-client --query "SYSTEM RELOAD USERS" - name: Run coverage tests run: | export SEQR_BASE_URL=https://test-seqr.org/ export NODE_ID=TestVLM - export CLICKHOUSE_SERVICE_HOSTNAME=clickhouse + export CLICKHOUSE_SERVICE_HOSTNAME=localhost export CLICKHOUSE_SERVICE_PORT=8123 export CLICKHOUSE_VLM_USERNAME=vlm_test_user export CLICKHOUSE_VLM_PASSWORD=vlm_test_password export CLICKHOUSE_DATABASE=test_seqr - coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py" -m pytest vlm/ + coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/conftest.py" -m pytest vlm/ coverage report -m --fail-under=95 diff --git a/reference_data/management/commands/update_all_reference_data.py b/reference_data/management/commands/update_all_reference_data.py index efbc9265ee..9a4f9cbe17 100644 --- a/reference_data/management/commands/update_all_reference_data.py +++ b/reference_data/management/commands/update_all_reference_data.py @@ -2,6 +2,7 @@ from collections import OrderedDict from django.core.management.base import BaseCommand, CommandError +from clickhouse_search.models.postgres_dicts import GeneIdDict from panelapp.models import PanelAppAU, PanelAppUK from reference_data.utils.gene_utils import get_genes_by_id_and_symbol from reference_data.models import GeneInfo, TranscriptInfo, HumanPhenotypeOntology, RefseqTranscript, GeneConstraint, \ @@ -57,6 +58,8 @@ def handle(self, *args, **options): data_model_name = GeneInfo.__name__ self._update_gencode(current_versions.get(data_model_name), options['gene_symbol_change_dir']) self._track_success_updates(data_model_name, latest_version, current_versions, updated) + GeneIdDict.reload() + gene_ids_to_gene, gene_symbols_to_gene = get_genes_by_id_and_symbol() if to_update else (None, None) for data_cls, latest_version in to_update.items(): diff --git a/reference_data/management/tests/update_all_reference_data_tests.py b/reference_data/management/tests/update_all_reference_data_tests.py index 32b0c83779..5f1bb2b263 100644 --- a/reference_data/management/tests/update_all_reference_data_tests.py +++ b/reference_data/management/tests/update_all_reference_data_tests.py @@ -42,6 +42,7 @@ def _mock_handler(_cls, **kwargs): class NewDbUpdateAllReferenceDataTest(BaseUpdateAllReferenceDataTest): + databases = '__all__' fixtures = ['users'] def test_empty_db_update_all_reference_data_command(self): @@ -78,6 +79,7 @@ def test_empty_db_update_all_reference_data_command(self): self.mock_slack.assert_not_called() self.assert_json_logs(user=None, expected=[ + ('Reloading dictionary seqrdb_gene_ids', None), ('unable to update PrimateAI: Primate_AI failed', { 'severity': 'ERROR', '@type': 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent', diff --git a/reference_data/management/tests/update_gencode_tests.py b/reference_data/management/tests/update_gencode_tests.py index e5b6ed3db1..82a1945508 100644 --- a/reference_data/management/tests/update_gencode_tests.py +++ b/reference_data/management/tests/update_gencode_tests.py @@ -54,6 +54,8 @@ class UpdateGencodeTest(ReferenceDataCommandTestCase): + databases = '__all__' + def setUp(self): super().setUp() @@ -202,6 +204,7 @@ def test_update_gencode_latest_command(self): ('Loaded 3 RefseqTranscript records', None), ('Skipped 1 records with unrecognized or duplicated transcripts', None), ('Updated GeneInfo reference data from version "31" to version "39"', None), + ('Reloading dictionary seqrdb_gene_ids', None), ('Done', None), ('Updated: GeneInfo', None), ]) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index d3684a5ef6..493bc8a380 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -5,6 +5,7 @@ from django.db.models import Count, Max, Q, F from django.db.models.fields.files import ImageFieldFile +from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict, IndividualMetadataDict from matchmaker.models import MatchmakerSubmission from reference_data.models import Omim from seqr.utils.gene_utils import get_genes_for_variant_display @@ -210,6 +211,10 @@ def delete_families_handler(request, project_guid): # delete families Family.bulk_delete(request.user, project=project, guid__in=family_guids_to_delete) + AffectedDict.reload(request.user) + SexDict.reload(request.user) + IndividualMetadataDict.reload(request.user) + # send response return create_json_response({ 'individualsByGuid': { @@ -230,9 +235,12 @@ def update_family_fields_handler(request, family_guid): request_json = json.loads(request.body) immutable_keys = [] if external_anvil_project_can_edit(family.project, request.user) else ['family_id'] + updated_fields = set() update_family_from_json(family, request_json, user=request.user, allow_unknown_keys=True, immutable_keys=[ 'display_name', - ] + immutable_keys) + ] + immutable_keys, updated_fields=updated_fields) + if updated_fields.intersection({'post_discovery_omim_numbers', 'post_discovery_mondo_id', 'analysis_status'}): + IndividualMetadataDict.reload(request.user) return create_json_response({ family.guid: _get_json_for_model(family, user=request.user, process_result=_set_display_name) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index 6a52e6eaec..0592396816 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -456,12 +456,19 @@ def test_update_success_story_types(self): data=json.dumps({'successStoryTypes': ['O', 'D']})) self.assertEqual(response.status_code, 403) + self.reset_logs() self.login_analyst_user() response = self.client.post(url, content_type='application/json', data=json.dumps({'successStoryTypes': ['O', 'D']})) self.assertEqual(response.status_code, 200) response_json = response.json() self.assertListEqual(response_json[FAMILY_GUID]['successStoryTypes'], ['O', 'D']) + self.assert_json_logs(self.analyst_user, [ + ('update Family F000001_1', {'dbUpdate': { + 'dbEntity': 'Family', 'entityId': 'F000001_1', 'updateType': 'update', 'updateFields': ['success_story_types'], + }}), + (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}), + ]) self.check_no_analyst_no_access(url, get_response=lambda: self.client.post( url, content_type='application/json', data=json.dumps({'successStoryTypes': []}))) @@ -471,6 +478,7 @@ def test_update_family_fields(self): url = reverse(update_family_fields_handler, args=[FAMILY_GUID]) self.check_collaborator_login(url) + self.reset_logs() body = {FAMILY_ID_FIELD: 'new_id', 'description': 'Updated description', 'analysis_status': 'C'} response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 200) @@ -481,6 +489,14 @@ def test_update_family_fields(self): self.assertEqual(response_json[FAMILY_GUID]['analysisStatus'], 'C') self.assertEqual(response_json[FAMILY_GUID]['analysisStatusLastModifiedBy'], 'Test Collaborator User') self.assertEqual(response_json[FAMILY_GUID]['analysisStatusLastModifiedDate'], '2020-01-01T00:00:00') + self.assert_json_logs(self.collaborator_user, [ + ('update Family F000001_1', {'dbUpdate': { + 'dbEntity': 'Family', 'entityId': 'F000001_1', 'updateType': 'update', + 'updateFields': ['analysis_status', 'analysis_status_last_modified_by', 'analysis_status_last_modified_date', 'description'], + }}), + ('Reloading dictionary seqrdb_individual_metadata_dict', None), + (None, {'httpRequest': mock.ANY, 'requestBody': body}), + ]) # Do not update audit fields if value does not change self.login_manager() diff --git a/seqr/views/apis/individual_api.py b/seqr/views/apis/individual_api.py index 06da8a78a8..9d7a0697b8 100644 --- a/seqr/views/apis/individual_api.py +++ b/seqr/views/apis/individual_api.py @@ -8,6 +8,7 @@ from django.contrib.auth.models import User from django.db.models import prefetch_related_objects +from clickhouse_search.models.postgres_dicts import IndividualMetadataDict, SexDict, AffectedDict from reference_data.models import HumanPhenotypeOntology from seqr.models import Individual, Family, CAN_VIEW from seqr.utils.file_utils import file_iter @@ -93,6 +94,7 @@ def update_individual_hpo_terms(request, individual_guid): for key in feature_fields } update_individual_from_json(individual, update_json, user=request.user, allow_features_update=True) + IndividualMetadataDict.reload(request.user) individual_json = {k: getattr(individual, _to_snake_case(k)) for k in feature_fields} add_individual_hpo_details([individual_json]) @@ -218,6 +220,9 @@ def delete_individuals_handler(request, project_guid): # delete the individuals families_with_deleted_individuals = delete_individuals(project, individual_guids_to_delete, request.user) + AffectedDict.reload(request.user) + SexDict.reload(request.user) + IndividualMetadataDict.reload(request.user) deleted_individuals_by_guid = { individual_guid: None for individual_guid in individual_guids_to_delete @@ -665,6 +670,9 @@ def save_individuals_metadata_table_handler(request, project_guid, upload_file_i if record.get(ASSIGNED_ANALYST_COL): family_assigned_analysts[record[ASSIGNED_ANALYST_COL]].append(individual.family.id) + if any(FEATURES_COL in record for record in json_records): + IndividualMetadataDict.reload(request.user) + response = { 'individualsByGuid': { individual['individualGuid']: individual for individual in _get_json_for_individuals( diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index a68a391116..910478d944 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -150,6 +150,7 @@ def test_update_individual_hpo_terms(self): edit_individuals_url = reverse(update_individual_hpo_terms, args=[INDIVIDUAL_UPDATE_GUID]) self.check_manager_login(edit_individuals_url) + self.reset_logs() response = self.client.post(edit_individuals_url, content_type='application/json', data=json.dumps(INDIVIDUAL_UPDATE_DATA)) @@ -176,6 +177,11 @@ def test_update_individual_hpo_terms(self): {'id': 'HP:0011675', 'notes': 'A new term'}, ]) + self.assert_json_logs(self.manager_user, [ + ('update Individual I000007_na20870', {'dbUpdate': mock.ANY}), + ('Reloading dictionary seqrdb_individual_metadata_dict', None), + ]) + @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP') @responses.activate def test_edit_individuals(self, mock_pm_group): @@ -378,6 +384,7 @@ def test_delete_individuals(self, mock_pm_group): dataset.active_individuals.set(set()) # send valid requests + self.reset_logs() response = self.client.post(individuals_url, content_type='application/json', data=json.dumps({ 'individuals': [INDIVIDUAL_IDS_UPDATE_DATA] })) @@ -392,6 +399,14 @@ def test_delete_individuals(self, mock_pm_group): self.assertFalse('I000002_na19678' in response_json['familiesByGuid']['F000001_1']['individualGuids']) self.assertIsNone(response_json['familiesByGuid']['F000001_1']['pedigreeImage']) self.assertFalse(Dataset.objects.filter(guid='S000130_na19678').exists()) + self.assert_json_logs(self.manager_user, [ + ('delete Dataset S000130_na19678', {'dbUpdate': mock.ANY}), + ('delete 1 Individuals', {'dbUpdate': mock.ANY}), + ('update 1 Familys', {'dbUpdate': mock.ANY}), + ('Reloading dictionary seqrdb_affected_status_dict', None), + ('Reloading dictionary seqrdb_sex_dict', None), + ('Reloading dictionary seqrdb_individual_metadata_dict', None), + ]) # Test PM permission pm_required_delete_individuals_url = reverse(delete_individuals_handler, args=[PM_REQUIRED_PROJECT_GUID]) @@ -577,6 +592,7 @@ def test_individuals_table_handler(self): url = reverse(save_individuals_table_handler, args=[PROJECT_GUID, response_json['uploadedFileId']]) + self.reset_logs() response = self.client.post(url) self.assertEqual(response.status_code, 200) response_json = response.json() @@ -611,6 +627,35 @@ def test_individuals_table_handler(self): self.assertEqual(response_json['individualsByGuid']['I000008_na20872']['individualId'], 'NA20872_update') self._assert_expected_reload_calls(PROJECT_GUID) + read_tmp_table_logs = self._read_tmp_table_logs('ffa788c23360f0908f3de16dca799fe1') + self.assert_json_logs(None, read_tmp_table_logs) + self.assert_json_logs(self.manager_user, [ + (mock.ANY, {'dbUpdate': { + 'dbEntity': 'Family', 'entityId': mock.ANY, 'updateFields': ['family_id', 'project'], 'updateType': 'create', + }}), + ('update Individual I000001_na19675', {'dbUpdate': mock.ANY}), + ('update Individual I000002_na19678', {'dbUpdate': mock.ANY}), + ('update Individual I000008_na20872', {'dbUpdate': mock.ANY}), + (mock.ANY, {'dbUpdate': { + 'dbEntity': 'Individual', 'entityId': mock.ANY, 'updateType': 'create', 'updateFields': [ + 'affected', 'case_review_status', 'family', 'individual_id', + ], + }}), + (mock.ANY, {'dbUpdate': { + 'dbEntity': 'FamilyNote', 'entityId': mock.ANY, 'updateType': 'create', 'updateFields': [ + 'family', 'note', 'note_type', + ], + }}), + (mock.ANY, {'dbUpdate': { + 'dbEntity': 'Individual', 'entityId': mock.ANY, 'updateType': 'update', 'updateFields': ['sex'], + }}), + ('update 3 Familys', {'dbUpdate': mock.ANY}), + ('Reloading dictionary seqrdb_affected_status_dict', None), + ('Triggering rebuild_gt_stats for R0001_1kg', None), + ('Triggered Rebuild Gt Stats', {'detail': {'project_guids': ['R0001_1kg']}}), + ('Reloading dictionary seqrdb_sex_dict', None), + ('Reloading dictionary seqrdb_individual_metadata_dict', None), + ], offset=len(read_tmp_table_logs)) # Test PM permission receive_url = reverse(receive_individuals_table_handler, args=[PM_REQUIRED_PROJECT_GUID]) @@ -633,6 +678,10 @@ def _assert_expected_reload_calls(self, project_guid): self.assertEqual(len(responses.calls), 1) self.assertDictEqual(json.loads(responses.calls[0].request.body), {'project_guids': [project_guid]}) + @staticmethod + def _read_tmp_table_logs(file_name): + return [] + @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') @mock.patch('seqr.views.utils.pedigree_info_utils.NO_VALIDATE_MANIFEST_PROJECT_CATEGORIES') @mock.patch('seqr.utils.communication_utils.EmailMultiAlternatives') @@ -1044,6 +1093,7 @@ def test_individuals_metadata_table_handler(self): ]}) # send valid request + self.reset_logs() header = f'family_id,{header}' rows[0] = '1,NA19678,,,,,false,,,,,' rows[1] = f'1,{rows[1]}' @@ -1053,6 +1103,15 @@ def test_individuals_metadata_table_handler(self): response = self.client.post(url, data={'f': f}) self._is_expected_individuals_metadata_upload(response, expected_families=True, has_non_hpo_update=True) + read_tmp_table_logs = self._read_tmp_table_logs('6aea3d1f1bfc295340aa504370102fd5') + offset = 2 if read_tmp_table_logs else 1 + self.assert_json_logs(None, read_tmp_table_logs, offset=offset) + self.assert_json_logs(self.collaborator_user, [ + ('update Individual I000002_na19678', {'dbUpdate': mock.ANY}), + ('update Individual I000001_na19675', {'dbUpdate': mock.ANY}), + ('Reloading dictionary seqrdb_individual_metadata_dict', None), + ], offset=offset+len(read_tmp_table_logs)) + def test_individuals_metadata_hpo_term_number_table_handler(self): url = reverse(receive_individuals_metadata_handler, args=['R0001_1kg']) self.check_collaborator_login(url) @@ -1468,3 +1527,10 @@ def _mock_subprocess(self, command, **kwargs): else: self.mock_subprocess.stdout.__iter__.return_value = self.gs_files[file_name] return self.mock_subprocess + + @staticmethod + def _read_tmp_table_logs(file_name): + return [ + (f'==> gsutil ls gs://seqr-scratch-temp/temp_upload_{file_name}.json.gz', None), + (f'==> gsutil cat gs://seqr-scratch-temp/temp_upload_{file_name}.json.gz | gunzip -c -q - ', None), + ] diff --git a/seqr/views/apis/project_api.py b/seqr/views/apis/project_api.py index db0eb69245..3585ae625e 100644 --- a/seqr/views/apis/project_api.py +++ b/seqr/views/apis/project_api.py @@ -7,6 +7,7 @@ from django.utils import timezone from notifications.models import Notification +from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict, IndividualMetadataDict from matchmaker.models import MatchmakerSubmission from seqr.models import Project, Family, Individual, Dataset, RnaSample, FamilyNote, PhenotypePrioritization, CAN_EDIT from seqr.utils.file_utils import file_iter @@ -59,6 +60,7 @@ def create_project_handler(request): project_args['is_mme_enabled'] = False project = create_model_from_json(Project, project_args, user=request.user) + IndividualMetadataDict.reload(request.user) return create_json_response({ 'projectsByGuid': { @@ -80,7 +82,7 @@ def update_project_handler(request, project_guid): check_project_permissions(project, request.user, can_edit=True) request_json = json.loads(request.body) - updated_fields = None + updated_fields = set() consent_code = request_json.get('consentCode') if consent_code and consent_code != project.consent_code: if not user_is_pm(request.user): @@ -89,6 +91,8 @@ def update_project_handler(request, project_guid): updated_fields = {'consent_code'} update_project_from_json(project, request_json, request.user, allow_unknown_keys=True, updated_fields=updated_fields) + if 'restrict_sharing' in updated_fields or 'vlm_contact_email' in updated_fields: + IndividualMetadataDict.reload(request.user) return create_json_response({ 'projectsByGuid': { @@ -378,6 +382,10 @@ def _delete_project(project_guid, user): project.delete_model(user, user_can_delete=True) + AffectedDict.reload(user) + SexDict.reload(user) + IndividualMetadataDict.reload(user) + if anvil_enabled() and not is_internal_anvil_project(project): AirtableSession(user, base=AirtableSession.ANVIL_BASE).safe_patch_records( ANVIL_REQUEST_TRACKING_TABLE, diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index bc13166699..c914cffb70 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -204,6 +204,7 @@ def test_update_project(self): self.assertEqual(project.genome_version, '37') self.assertEqual(project.consent_code, 'H') + self.reset_logs() response = self.client.post(update_project_url, content_type='application/json', data=json.dumps( {'description': 'updated project description', 'restrictSharing': True, 'genomeVersion': '38', 'workspaceName': 'test update name'} )) @@ -218,6 +219,14 @@ def test_update_project(self): self.assertEqual(updated_project.description, 'updated project description') self.assertEqual(updated_project.genome_version, '37') self.assertEqual(updated_project.workspace_name, expected_workspace_name) + self.assert_json_logs(self.manager_user, [ + ('update Project R0001_1kg', {'dbUpdate': { + 'dbEntity': 'Project', 'entityId': 'R0001_1kg', 'updateType': 'update', + 'updateFields': ['description', 'restrict_sharing'], + }}), + ('Reloading dictionary seqrdb_individual_metadata_dict', None), + (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}), + ]) # test consent code response = self.client.post(update_project_url, content_type='application/json', data=json.dumps( @@ -225,12 +234,19 @@ def test_update_project(self): )) self.assertEqual(response.status_code, 403) self.login_pm_user() + self.reset_logs() response = self.client.post(update_project_url, content_type='application/json', data=json.dumps( {'consentCode': 'G'} )) self.assertEqual(response.status_code, 200) self.assertEqual(response.json()['projectsByGuid'][PROJECT_GUID]['consentCode'], 'G') self.assertEqual(Project.objects.get(guid=PROJECT_GUID).consent_code, 'G') + self.assert_json_logs(self.pm_user, [ + ('update Project R0001_1kg', {'dbUpdate': { + 'dbEntity': 'Project', 'entityId': 'R0001_1kg', 'updateType': 'update', 'updateFields': ['consent_code'], + }}), + (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}), + ]) @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', None) def test_create_project_no_pm(self): diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 2edbc90742..3e46cf9680 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -1331,10 +1331,18 @@ def test_family_metadata(self): self.assertListEqual( sorted([r['familyGuid'] for r in response.json()['rows']]), expected_families + self.ADDITIONAL_FAMILIES) + @responses.activate def test_variant_metadata(self): url = reverse(variant_metadata, args=[PROJECT_GUID]) self.check_analyst_login(url) + responses.add(responses.GET, 'https://monarchinitiative.org/v3/api/entity/MONDO:0044970', status=200, json={ + 'id': 'MONDO:0044970', + 'category': 'biolink:Disease', + 'name': 'mitochondrial disease', + 'inheritance': {}, + }) + response = self.client.get(url) self.assertEqual(response.status_code, 200) response_json = response.json() diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 099d210914..68f99caf28 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -3,6 +3,7 @@ from django.db.models import Q +from clickhouse_search.models.postgres_dicts import DiscoveryVariantDict, ExcludedVariantDict from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\ Family, GeneNote, Project, Dataset from seqr.utils.xpos_utils import get_xpos @@ -13,7 +14,7 @@ get_json_for_saved_variants_child_entities, get_json_for_gene_notes_by_gene_id, STRUCTURED_METADATA_TAG_TYPES from seqr.views.utils.permissions_utils import get_project_and_check_permissions, check_project_permissions, \ login_and_policies_required -from seqr.views.utils.variant_utils import get_variants_response, parse_saved_variant_json +from seqr.views.utils.variant_utils import get_variants_response, parse_saved_variant_json, DISCOVERY_CATEGORY logger = logging.getLogger(__name__) @@ -274,9 +275,9 @@ def _update_variant_tag_models(request, variant_guids, tag_key, model_cls, get_t tag_type = tag_key.lower().rstrip('s') updated_data = request_json.get(tag_key, []) - deleted_guids = _delete_removed_tags(saved_variants, all_variant_guids, updated_data, request.user, tag_type, protected_tag_types) + deleted_guids, has_discovery_update, has_excluded_update = _delete_removed_tags(saved_variants, all_variant_guids, updated_data, request.user, tag_type, protected_tag_types) - _update_tags(saved_variants, request_json, request.user, tag_key, model_cls, get_tag_create_data) + _update_tags(saved_variants, request_json, request.user, tag_key, model_cls, get_tag_create_data, has_discovery_update=has_discovery_update, has_excluded_update=has_excluded_update) saved_variant_id_map = {sv.id: sv.guid for sv in saved_variants} tags, variant_tag_map = get_json_for_saved_variants_child_entities(model_cls, saved_variant_id_map) @@ -311,15 +312,23 @@ def _delete_removed_tags(saved_variants, all_variant_guids, tag_updates, user, t remove_tags = tag_set.exclude(guid__in=existing_tag_guids) if protected_tag_types: remove_tags = remove_tags.exclude(variant_tag_type__name__in=protected_tag_types) + track_updates = tag_type == 'tag' + has_discovery_update = False + has_excluded_update = False for tag in remove_tags: tag_variant_guids = {sv.guid for sv in tag.saved_variants.all()} if tag_variant_guids == all_variant_guids: + if track_updates: + if tag.variant_tag_type.category == DISCOVERY_CATEGORY: + has_discovery_update = True + elif tag.variant_tag_type.name == 'Excluded': + has_excluded_update = True deleted_tag_guids.append(tag.guid) tag.delete_model(user, user_can_delete=True) - return deleted_tag_guids + return deleted_tag_guids, has_discovery_update, has_excluded_update -def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=VariantTag, get_tag_create_data=_get_tag_type_create_data): +def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=VariantTag, get_tag_create_data=_get_tag_type_create_data, has_discovery_update=False, has_excluded_update=False): tags = tags_json.get(tag_key, []) for tag in tags: if tag.get('tagGuid'): @@ -327,6 +336,11 @@ def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=Vari update_model_from_json(model, tag, user=user, allow_unknown_keys=True) else: create_data = get_tag_create_data(tag, saved_variants=saved_variants) + if 'variant_tag_type' in create_data: + if create_data['variant_tag_type'].category == DISCOVERY_CATEGORY: + has_discovery_update = True + elif create_data['variant_tag_type'].name == 'Excluded': + has_excluded_update = True create_data.update({ 'metadata': tag.get('metadata'), 'search_hash': tags_json.get('searchHash'), @@ -334,6 +348,11 @@ def _update_tags(saved_variants, tags_json, user, tag_key='tags', model_cls=Vari model = create_model_from_json(model_cls, create_data, user) model.saved_variants.set(saved_variants) + if has_discovery_update: + DiscoveryVariantDict.reload(user) + if has_excluded_update: + ExcludedVariantDict.reload(user) + @login_and_policies_required def update_variant_main_transcript(request, variant_guid, transcript_id): diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 954ecb7d03..c13cca28f2 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -3,15 +3,15 @@ from django.urls.base import reverse -from clickhouse_search.models.reference_data_models import DbnsfpGRCh37SnvIndelMv, DbnsfpGRCh37SnvIndelDict +from clickhouse_search.all_search_tests import ClickhouseSearchTestCase from seqr.models import SavedVariant, VariantNote, VariantTag, VariantFunctionalData, Project from seqr.views.apis.saved_variant_api import saved_variant_data, create_variant_note_handler, create_saved_variant_handler, \ update_variant_note_handler, delete_variant_note_handler, update_variant_tags_handler, create_manual_saved_variant_handler, \ update_variant_main_transcript, update_variant_functional_data_handler, update_variant_acmg_classification_handler from seqr.views.utils.orm_to_json_utils import get_json_for_saved_variants -from seqr.views.utils.test_utils import AuthenticationTestCase, SAVED_VARIANT_DETAIL_FIELDS, TAG_FIELDS, GENE_VARIANT_FIELDS, \ +from seqr.views.utils.test_utils import SAVED_VARIANT_DETAIL_FIELDS, TAG_FIELDS, GENE_VARIANT_FIELDS, \ TAG_TYPE_FIELDS, LOCUS_LIST_FIELDS, PA_LOCUS_LIST_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \ - FAMILY_NOTE_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, SAVED_VARIANT_FIELDS, AnvilAuthenticationTestCase + FAMILY_NOTE_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, SAVED_VARIANT_FIELDS PROJECT_GUID = 'R0001_1kg' @@ -144,10 +144,13 @@ } -class SavedVariantAPITest(object): +class SavedVariantAPITest(ClickhouseSearchTestCase): + fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'clickhouse_discovery_variants', 'clickhouse_saved_variants'] @mock.patch('seqr.views.utils.variant_utils.OMIM_GENOME_VERSION', '37') def test_saved_variant_data(self): + Project.objects.filter(guid__in=[PROJECT_GUID, 'R0003_test']).update(genome_version='37') + url = reverse(saved_variant_data, args=[PROJECT_GUID]) self.check_collaborator_login(url) @@ -378,6 +381,17 @@ def test_saved_variant_data(self): *variant_fields, *SAVED_VARIANT_DETAIL_FIELDS, 'discoveryTags', 'noAccessDiscoveryFamilies', 'screenRegionType', 'sortedRegulatoryFeatureConsequences', 'sortedMotifFeatureConsequences', }) + self.mock_list_workspaces.assert_called_with(self.analyst_user) + self.mock_get_ws_access_level.assert_any_call( + mock.ANY, 'ext-data', 'empty') + self.mock_get_ws_access_level.assert_called_with( + mock.ANY, 'my-seqr-billing', 'anvil-1kg project n\u00e5me with uni\u00e7\u00f8de') + self.assertEqual(self.mock_get_ws_access_level.call_count, 15) + self.mock_get_groups.assert_called_with(self.collaborator_user) + self.assertEqual(self.mock_get_groups.call_count, 1) + self.mock_get_ws_acl.assert_not_called() + self.mock_get_group_members.assert_not_called() + def test_create_saved_variant(self): create_saved_variant_url = reverse(create_saved_variant_handler) self.check_collaborator_login(create_saved_variant_url, request_data={'familyGuid': 'F000001_1'}) @@ -430,6 +444,8 @@ def test_create_saved_variant(self): self.assertEqual(response.status_code, 200) self.assertListEqual(list(response.json()['savedVariantsByGuid'].keys()), [variant_guid]) + self.assert_no_list_ws_has_al(3) + def _assert_created_variant(self, saved_variant, variant_json, gene_ids=None, dataset_type='SNV_INDEL', sv_type=None, main_transcript=None): for field in ['xpos', 'ref', 'alt', 'key']: self.assertEqual(variant_json.get(field), getattr(saved_variant, field, None)) @@ -511,6 +527,8 @@ def test_create_saved_sv_variant(self): self.assertDictEqual(response_json['variantTagsByGuid'], {}) self.assertDictEqual(response_json['variantFunctionalDataByGuid'], {}) + self.assert_no_list_ws_has_al(2) + def test_create_saved_compound_hets(self): create_saved_compound_hets_url = reverse(create_saved_variant_handler) self.check_collaborator_login(create_saved_compound_hets_url, request_data={'familyGuid': 'F000001_1'}) @@ -569,6 +587,8 @@ def test_create_saved_compound_hets(self): self.assertListEqual(["Review"], [vt.variant_tag_type.name for vt in VariantTag.objects.filter( saved_variants__guid__contains=new_compound_het_4_guid)]) + self.assert_no_list_ws_has_al(2) + def test_create_manual_variant(self): create_saved_variant_url = reverse(create_manual_saved_variant_handler, args=['F000001_1']) self.check_collaborator_login(create_saved_variant_url) @@ -766,6 +786,8 @@ def test_create_update_and_delete_variant_note(self): new_variant_note = VariantNote.objects.filter(guid=updated_note_response['noteGuid']) self.assertEqual(len(new_variant_note), 0) + self.assert_no_list_ws_has_al(8) + def test_create_partially_saved_compound_het_variant_note(self): # compound het 5 is not saved, whereas compound het 1 is saved create_saved_variant_url = reverse(create_saved_variant_handler) @@ -812,6 +834,8 @@ def test_create_partially_saved_compound_het_variant_note(self): 'one_saved_one_not_saved_compount_hets_note', response_json['variantNotesByGuid'][note_guids[0]]['note'], ) + self.assert_no_list_ws_has_al(2) + def test_create_update_and_delete_compound_hets_variant_note(self): # send valid request to create variant_note for compound hets create_comp_hets_variant_note_url = reverse(create_variant_note_handler, args=[','.join([COMPOUND_HET_1_GUID, COMPOUND_HET_2_GUID])]) @@ -916,6 +940,8 @@ def test_create_update_and_delete_compound_hets_variant_note(self): variants = SavedVariant.objects.filter(guid__in=[COMPOUND_HET_1_GUID, COMPOUND_HET_2_GUID]) self.assertEqual(len(variants), 0) + self.assert_no_list_ws_has_al(7) + def test_update_variant_tags(self): variant_tags = VariantTag.objects.filter(saved_variants__guid__contains=VARIANT_GUID) self.assertSetEqual({"Review", "Tier 1 - Novel gene and phenotype"}, {vt.variant_tag_type.name for vt in variant_tags}) @@ -923,6 +949,7 @@ def test_update_variant_tags(self): update_variant_tags_url = reverse(update_variant_tags_handler, args=[VARIANT_GUID]) self.check_collaborator_login(update_variant_tags_url, request_data={'familyGuid': 'F000001_1'}) + self.reset_logs() review_guid = 'VT1708633_2103343353_r0390_100' response = self.client.post(update_variant_tags_url, content_type='application/json', data=json.dumps({ 'tags': [ @@ -945,8 +972,19 @@ def test_update_variant_tags(self): self.assertSetEqual( {"Review", "Excluded"}, {vt.variant_tag_type.name for vt in VariantTag.objects.filter(saved_variants__guid__contains=VARIANT_GUID)}) + self.assert_json_logs(self.collaborator_user, [ + ('delete VariantTag VT1726961_2103343353_r0390_100', {'dbUpdate': mock.ANY}), + ('update VariantTag VT1708633_2103343353_r0390_100', {'dbUpdate': mock.ANY}), + (mock.ANY, {'dbUpdate': {'dbEntity': 'VariantTag', 'entityId': mock.ANY, 'updateType': 'create', 'updateFields': [ + 'metadata', 'search_hash', 'variant_tag_type', + ]}}), + ('Reloading dictionary seqrdb_discovery_variant_dict', None), + ('Reloading dictionary seqrdb_excluded_variant_dict', None), + (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}), + ]) # test delete all - with MME submission + self.reset_logs() response = self.client.post(update_variant_tags_url, content_type='application/json', data=json.dumps({ 'tags': [], 'familyGuid': 'F000001_1' @@ -958,6 +996,12 @@ def test_update_variant_tags(self): }) self.assertEqual(VariantTag.objects.filter(saved_variants__guid__contains=VARIANT_GUID).count(), 0) self.assertEqual(SavedVariant.objects.filter(guid=VARIANT_GUID).count(), 1) + self.assert_json_logs(self.collaborator_user, [ + (mock.ANY, {'dbUpdate': {'dbEntity': 'VariantTag', 'entityId': mock.ANY, 'updateType': 'delete'}}), + (mock.ANY, {'dbUpdate': {'dbEntity': 'VariantTag', 'entityId': mock.ANY, 'updateType': 'delete'}}), + ('Reloading dictionary seqrdb_excluded_variant_dict', None), + (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}), + ]) # test delete all - no MME submission update_no_submission_variant_tags_url = reverse(update_variant_tags_handler, args=[COMPOUND_HET_1_GUID]) @@ -972,6 +1016,8 @@ def test_update_variant_tags(self): self.assertEqual(VariantTag.objects.filter(saved_variants__guid__contains=COMPOUND_HET_1_GUID).count(), 0) self.assertEqual(SavedVariant.objects.filter(guid=COMPOUND_HET_1_GUID).count(), 0) + self.assert_no_list_ws_has_al(4) + def test_update_variant_functional_data(self): variant_functional_data = VariantFunctionalData.objects.filter(saved_variants__guid__contains=VARIANT_GUID) self.assertSetEqual( @@ -1010,6 +1056,8 @@ def test_update_variant_functional_data(self): {vt.functional_data_tag for vt in variant_functional_data}) self.assertSetEqual({"An updated note", "0.05"}, {vt.metadata for vt in variant_functional_data}) + self.assert_no_list_ws_has_al(2) + def test_update_compound_hets_variant_tags(self): variant_tags = VariantTag.objects.filter(saved_variants__guid__in=[COMPOUND_HET_1_GUID, COMPOUND_HET_2_GUID]) self.assertEqual(len(variant_tags), 0) @@ -1018,6 +1066,7 @@ def test_update_compound_hets_variant_tags(self): update_variant_tags_handler, args=[','.join([COMPOUND_HET_1_GUID, COMPOUND_HET_2_GUID])]) self.check_collaborator_login(update_variant_tags_url, request_data={'familyGuid': 'F000001_1'}) + self.reset_logs() response = self.client.post(update_variant_tags_url, content_type='application/json', data=json.dumps({ 'tags': [{'name': 'Review'}, {'name': 'Excluded'}], 'familyGuid': 'F000001_1' @@ -1042,6 +1091,17 @@ def test_update_compound_hets_variant_tags(self): {vt.variant_tag_type.name for vt in VariantTag.objects.filter( saved_variants__guid__in=[COMPOUND_HET_1_GUID, COMPOUND_HET_2_GUID])}) + self.assert_json_logs(self.collaborator_user, [ + (mock.ANY, {'dbUpdate': {'dbEntity': 'VariantTag', 'entityId': mock.ANY, 'updateType': 'create', 'updateFields': [ + 'metadata', 'search_hash', 'variant_tag_type', + ]}}), + (mock.ANY, {'dbUpdate': {'dbEntity': 'VariantTag', 'entityId': mock.ANY, 'updateType': 'create', 'updateFields': [ + 'metadata', 'search_hash', 'variant_tag_type', + ]}}), + ('Reloading dictionary seqrdb_excluded_variant_dict', None), + (None, {'httpRequest': mock.ANY, 'requestBody': mock.ANY}), + ]) + invalid_url = reverse(update_variant_tags_handler, args=['not_variant,{}'.format(COMPOUND_HET_1_GUID)]) response = self.client.post(invalid_url, content_type='application/json', data=json.dumps({ 'tags': [{'name': 'Review'}, {'name': 'Excluded'}], @@ -1050,6 +1110,8 @@ def test_update_compound_hets_variant_tags(self): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': 'Unable to find the following variant(s): not_variant'}) + self.assert_no_list_ws_has_al(3) + def test_update_compound_hets_variant_functional_data(self): variant_functional_data = VariantFunctionalData.objects.filter( saved_variants__guid__in=[COMPOUND_HET_1_GUID, COMPOUND_HET_2_GUID]) @@ -1095,6 +1157,8 @@ def test_update_compound_hets_variant_functional_data(self): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': 'Unable to find the following variant(s): not_variant'}) + self.assert_no_list_ws_has_al(3) + def test_update_variant_main_transcript(self): transcript_id = 'ENST00000438943' update_main_transcript_url = reverse(update_variant_main_transcript, args=[VARIANT_GUID, transcript_id]) @@ -1111,6 +1175,8 @@ def test_update_variant_main_transcript(self): self.assertDictEqual(saved_variants.first().main_transcript, transcript) self.assertEqual(get_json_for_saved_variants(saved_variants)[0]['selectedMainTranscriptId'], transcript_id) + self.assert_no_list_ws_has_al(2) + def test_update_variant_acmg_classification(self): update_variant_acmg_classification_url = reverse(update_variant_acmg_classification_handler, args=[VARIANT_GUID]) self.check_collaborator_login(update_variant_acmg_classification_url) @@ -1129,87 +1195,11 @@ def test_update_variant_acmg_classification(self): self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), {'savedVariantsByGuid': {VARIANT_GUID: {'acmgClassification': variant['variant']['acmgClassification']}}}) + self.assert_no_list_ws_has_al(2) -# Tests for AnVIL access disabled -class LocalSavedVariantAPITest(AuthenticationTestCase, SavedVariantAPITest): - fixtures = ['users', '1kg_project', 'reference_data', 'clickhouse_discovery_variants', 'clickhouse_saved_variants'] - - -def assert_no_list_ws_has_al(self, acl_call_count): - self.mock_list_workspaces.assert_not_called() - self.mock_get_ws_access_level.assert_called_with(mock.ANY, - 'my-seqr-billing', 'anvil-1kg project n\u00e5me with uni\u00e7\u00f8de') - self.assertEqual(self.mock_get_ws_access_level.call_count, acl_call_count) - self.assert_no_extra_anvil_calls() - - -# Test for permissions from AnVIL only -class AnvilSavedVariantAPITest(AnvilAuthenticationTestCase, SavedVariantAPITest): - fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'clickhouse_discovery_variants', 'clickhouse_saved_variants'] - - @classmethod - def setUpTestData(cls): - super().setUpTestData() - DbnsfpGRCh37SnvIndelMv.refresh() - DbnsfpGRCh37SnvIndelDict.reload() - - def test_saved_variant_data(self, *args): - super(AnvilSavedVariantAPITest, self).test_saved_variant_data(*args) - self.mock_list_workspaces.assert_called_with(self.analyst_user) - self.mock_get_ws_access_level.assert_any_call( - mock.ANY, 'ext-data', 'empty') - self.mock_get_ws_access_level.assert_called_with( - mock.ANY, 'my-seqr-billing', 'anvil-1kg project n\u00e5me with uni\u00e7\u00f8de') - self.assertEqual(self.mock_get_ws_access_level.call_count, 15) - self.mock_get_groups.assert_called_with(self.collaborator_user) - self.assertEqual(self.mock_get_groups.call_count, 1) - self.mock_get_ws_acl.assert_not_called() - self.mock_get_group_members.assert_not_called() - - def test_create_saved_variant(self): - super(AnvilSavedVariantAPITest, self).test_create_saved_variant() - assert_no_list_ws_has_al(self, 3) - - def test_create_saved_sv_variant(self): - super(AnvilSavedVariantAPITest, self).test_create_saved_sv_variant() - assert_no_list_ws_has_al(self, 2) - - def test_create_saved_compound_hets(self): - super(AnvilSavedVariantAPITest, self).test_create_saved_compound_hets() - assert_no_list_ws_has_al(self, 2) - - def test_create_update_and_delete_variant_note(self): - super(AnvilSavedVariantAPITest, self).test_create_update_and_delete_variant_note() - assert_no_list_ws_has_al(self, 8) - - def test_create_partially_saved_compound_het_variant_note(self): - super(AnvilSavedVariantAPITest, self).test_create_partially_saved_compound_het_variant_note() - assert_no_list_ws_has_al(self, 2) - - def test_create_update_and_delete_compound_hets_variant_note(self): - super(AnvilSavedVariantAPITest, self).test_create_update_and_delete_compound_hets_variant_note() - assert_no_list_ws_has_al(self, 7) - - def test_update_variant_tags(self): - super(AnvilSavedVariantAPITest, self).test_update_variant_tags() - assert_no_list_ws_has_al(self, 4) - - def test_update_variant_functional_data(self): - super(AnvilSavedVariantAPITest, self).test_update_variant_functional_data() - assert_no_list_ws_has_al(self, 2) - - def test_update_compound_hets_variant_tags(self): - super(AnvilSavedVariantAPITest, self).test_update_compound_hets_variant_tags() - assert_no_list_ws_has_al(self, 3) - - def test_update_compound_hets_variant_functional_data(self): - super(AnvilSavedVariantAPITest, self).test_update_compound_hets_variant_functional_data() - assert_no_list_ws_has_al(self, 3) - - def test_update_variant_main_transcript(self): - super(AnvilSavedVariantAPITest, self).test_update_variant_main_transcript() - assert_no_list_ws_has_al(self, 2) - - def test_update_variant_acmg_classification(self): - super(AnvilSavedVariantAPITest, self).test_update_variant_acmg_classification() - assert_no_list_ws_has_al(self, 2) + def assert_no_list_ws_has_al(self, acl_call_count): + self.mock_list_workspaces.assert_not_called() + self.mock_get_ws_access_level.assert_called_with(mock.ANY, + 'my-seqr-billing', 'anvil-1kg project n\u00e5me with uni\u00e7\u00f8de') + self.assertEqual(self.mock_get_ws_access_level.call_count, acl_call_count) + self.assert_no_extra_anvil_calls() diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index ea21b57f45..e396c365e9 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -1,6 +1,6 @@ from collections import defaultdict -from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict +from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict, IndividualMetadataDict from matchmaker.models import MatchmakerSubmission, MatchmakerResult from seqr.models import Dataset, IgvSample, RnaSample, Individual, Family, FamilyNote from seqr.utils.middleware import ErrorsWarningsException @@ -25,6 +25,7 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge updated_individuals = set() updated_affected = set() updated_sex = set() + updated_metadata = set() updated_note_ids = [] parent_updates = [] num_created_families = 0 @@ -56,7 +57,7 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge for record in individual_records: created_individual = _update_from_record( - record, user, families_by_id, individual_lookup, updated_family_ids, updated_individuals, updated_affected, updated_sex, parent_updates, updated_note_ids, allow_features_update) + record, user, families_by_id, individual_lookup, updated_family_ids, updated_individuals, updated_affected, updated_sex, updated_metadata, parent_updates, updated_note_ids, allow_features_update) if created_individual: num_created_individuals += 1 @@ -70,12 +71,14 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge updated_family_models = Family.objects.filter(id__in=updated_family_ids) _remove_pedigree_images(updated_family_models, user) - if updated_affected: + if updated_affected or num_created_individuals > 0: AffectedDict.reload(user) - if not skip_gt_stats_rebuild: + if updated_affected and not skip_gt_stats_rebuild: trigger_rebuild_gt_stats(project, user) - if updated_sex: + if updated_sex or num_created_individuals > 0: SexDict.reload(user) + if updated_metadata or num_created_families > 0 or num_created_individuals > 0: + IndividualMetadataDict.reload(user) pedigree_json = None if get_update_json: @@ -90,7 +93,7 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge return pedigree_json -def _update_from_record(record, user, families_by_id, individual_lookup, updated_family_ids, updated_individuals, updated_affected, updated_sex, parent_updates, updated_note_ids, allow_features_update): +def _update_from_record(record, user, families_by_id, individual_lookup, updated_family_ids, updated_individuals, updated_affected, updated_sex, updated_metadata, parent_updates, updated_note_ids, allow_features_update): family_id = _get_record_family_id(record) family = families_by_id.get(family_id) created_individual = False @@ -153,6 +156,8 @@ def _update_from_record(record, user, families_by_id, individual_lookup, updated updated_affected.add(individual) if 'sex' in updated_fields: updated_sex.add(individual) + if 'features' in updated_fields: + updated_metadata.add(individual) if family.pedigree_image: updated_family_ids.add(family.id) diff --git a/seqr/views/utils/json_to_orm_utils.py b/seqr/views/utils/json_to_orm_utils.py index 02a65aa805..909c8b778b 100644 --- a/seqr/views/utils/json_to_orm_utils.py +++ b/seqr/views/utils/json_to_orm_utils.py @@ -15,14 +15,14 @@ def update_project_from_json(project, json, user, allow_unknown_keys=False, upda immutable_keys=['consent_code', 'genome_version', 'workspace_namespace', 'workspace_name']) -def update_family_from_json(family, json, user, allow_unknown_keys=False, immutable_keys=None): +def update_family_from_json(family, json, user, allow_unknown_keys=False, immutable_keys=None, updated_fields=None): if json.get('displayName') and json['displayName'] == family.family_id: json['displayName'] = '' immutable_keys = (immutable_keys or []) + ['pedigree_image', 'assigned_analyst', 'case_review_summary', 'case_review_notes', 'guid'] return update_model_from_json( - family, json, user=user, allow_unknown_keys=allow_unknown_keys, immutable_keys=immutable_keys, + family, json, user=user, allow_unknown_keys=allow_unknown_keys, immutable_keys=immutable_keys, updated_fields=updated_fields, ) diff --git a/settings.py b/settings.py index 5c3871d8cf..7c21b1584e 100644 --- a/settings.py +++ b/settings.py @@ -234,36 +234,37 @@ 'USER': os.environ.get('POSTGRES_USERNAME', 'postgres'), 'PASSWORD': os.environ.get('POSTGRES_PASSWORD', 'pgtest'), } +CLICKHOUSE_DB_CONFIG = { + 'ENGINE': 'clickhouse_search.backend', + 'NAME': 'seqr', + 'HOST': os.environ.get('CLICKHOUSE_SERVICE_HOSTNAME', 'localhost'), + 'PORT': int(os.environ.get('CLICKHOUSE_SERVICE_PORT', '9000')), + 'OPTIONS': { + 'settings': { + 'use_client_time_zone': False, + } + } +} +CLICKHOUSE_WRITER_USER = os.environ.get('CLICKHOUSE_WRITER_USER', 'clickhouse') +CLICKHOUSE_WRITER_PASSWORD = os.environ.get('CLICKHOUSE_WRITER_PASSWORD', 'clickhouse_test') DATABASES = { 'default': dict(NAME='seqrdb', **POSTGRES_DB_CONFIG), 'reference_data': dict(NAME='reference_data_db', **POSTGRES_DB_CONFIG), + 'clickhouse_write': { + **CLICKHOUSE_DB_CONFIG, + 'USER': CLICKHOUSE_WRITER_USER, + 'PASSWORD': CLICKHOUSE_WRITER_PASSWORD, + }, + 'clickhouse': { + **CLICKHOUSE_DB_CONFIG, + 'USER': os.environ.get('CLICKHOUSE_READER_USER', 'clickhouse'), + 'PASSWORD': os.environ.get('CLICKHOUSE_READER_PASSWORD', 'clickhouse_test'), + }, } DATABASE_ROUTERS = ['reference_data.models.ReferenceDataRouter', 'clickhouse_search.models.ClickHouseRouter'] CLICKHOUSE_IN_MEMORY_DIR = os.environ.get('CLICKHOUSE_IN_MEMORY_DIR', '/in-memory-dir') CLICKHOUSE_DATA_DIR = os.getenv('CLICKHOUSE_DATA_DIR', '/var/seqr/clickhouse-data') -CLICKHOUSE_SERVICE_HOSTNAME = os.environ.get('CLICKHOUSE_SERVICE_HOSTNAME') -CLICKHOUSE_WRITER_USER = os.environ.get('CLICKHOUSE_WRITER_USER', 'clickhouse') -CLICKHOUSE_WRITER_PASSWORD = os.environ.get('CLICKHOUSE_WRITER_PASSWORD', 'clickhouse_test') -if CLICKHOUSE_SERVICE_HOSTNAME: - DATABASES['clickhouse_write'] = { - 'ENGINE': 'clickhouse_search.backend', - 'NAME': 'seqr', - 'HOST': CLICKHOUSE_SERVICE_HOSTNAME, - 'PORT': int(os.environ.get('CLICKHOUSE_SERVICE_PORT', '9000')), - 'USER': CLICKHOUSE_WRITER_USER, - 'PASSWORD': CLICKHOUSE_WRITER_PASSWORD, - 'OPTIONS': { - 'settings': { - 'use_client_time_zone': False, - } - }, - } - DATABASES['clickhouse'] = { - **DATABASES['clickhouse_write'], - 'USER': os.environ.get('CLICKHOUSE_READER_USER', 'clickhouse'), - 'PASSWORD': os.environ.get('CLICKHOUSE_READER_PASSWORD', 'clickhouse_test'), - } TEST_RUNNER = "seqr.testrunner.OrderedDatabaseDeletionRunner" diff --git a/ui/create_jsx_test.py b/ui/create_jsx_test.py deleted file mode 100644 index d6deeaf1f1..0000000000 --- a/ui/create_jsx_test.py +++ /dev/null @@ -1,85 +0,0 @@ -import argparse as ap -import re -import os - -p = ap.ArgumentParser("Create a .test.js file with a shallow-render test template for a .jsx component. " - "This script expects the .jsx file to export the class it defines using a statement like:" - "'export { SomeWidget as SomeWidgetComponent }'" ) - -p.add_argument('jsx_path', help='Path of .jsx component') -args = p.parse_args() - -jsx_dir = os.path.dirname(args.jsx_path) -jsx_filename = os.path.basename(args.jsx_path) - -outf = open(os.path.join(jsx_dir, jsx_filename.replace('.jsx', '.test.js')), 'w') - -class_name, component_name = None, None -propTypes = "" -mapStateToProps = "" -imports = "" -f = open(args.jsx_path) - -for line in f: - # match lines like 'export { FilterSelector as FilterSelectorComponent }' - match = re.match('export[ ]* \{[ ]*([^ ]+)[ ]* as [ ]*([^ ]+)[ ]*\}', line) - if match: - class_name = match.group(1) - component_name = match.group(2) - - match = re.match('class[ ]+([^ ]+)[ ]+extends[ ]+React.Component', line) - if match: - class_name = component_name = match.group(1) - - match = re.match('const[ ]+([^ ]+)[ ]+=[ (]+props[ )]+=>', line) - if match: - class_name = component_name = match.group(1) - - elif line.startswith('import') and 'rootReducer' in line: - imports += line - - elif "propTypes =" in line: - while '}' not in line: - line = next(f) - if ':' in line: - propTypes += ' ' + line.strip()+'\n' - propTypes = propTypes.rstrip('\n') - - elif "mapStateToProps =" in line: - while '}' not in line: - line = next(f) - if ':' in line: - mapStateToProps += ' ' + line.strip()+'\n' - mapStateToProps = mapStateToProps.replace('state', 'STATE1').rstrip('\n') - -if not class_name: - #p.error("export line (eg. 'export { FilterSelector as FilterSelectorComponent }') not found") - class_name = component_name = os.path.basename(args.jsx_path).replace('.jsx', '') - -if not mapStateToProps: - mapStateToProps = propTypes - - -template = """import React from 'react' -import { shallow, configure } from 'enzyme' -import Adapter from 'enzyme-adapter-react-16' -import %(component_name)s from './%(class_name)s' -%(imports)s - -configure({ adapter: new Adapter() }) - -test('shallow-render without crashing', () => { - /* -%(propTypes)s - */ - - const props = { -%(mapStateToProps)s - } - - shallow(<%(component_name)s {...props} />) -}) -""" % locals() - -outf.write(template) -outf.close() diff --git a/ui/package-lock.json b/ui/package-lock.json index 94c0415ba7..cbbc6ace05 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -16113,10 +16113,13 @@ } }, "node_modules/shell-quote": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.1.tgz", - "integrity": "sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==", + "version": "1.8.4", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.4.tgz", + "integrity": "sha512-VsC6n6vz1ihYYyZZwX7YZSF5l5x36ca17OC+a69h94YqB7X6XLwf+5MOgynYir2SLFUbl8gIYvBo8K8RoNQ6bQ==", "dev": true, + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -31977,9 +31980,9 @@ "dev": true }, "shell-quote": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.1.tgz", - "integrity": "sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==", + "version": "1.8.4", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.4.tgz", + "integrity": "sha512-VsC6n6vz1ihYYyZZwX7YZSF5l5x36ca17OC+a69h94YqB7X6XLwf+5MOgynYir2SLFUbl8gIYvBo8K8RoNQ6bQ==", "dev": true }, "side-channel": { diff --git a/ui/shared/components/panel/family/FamilyReads.jsx b/ui/shared/components/panel/family/FamilyReads.jsx index 70d6a41a55..bd95805019 100644 --- a/ui/shared/components/panel/family/FamilyReads.jsx +++ b/ui/shared/components/panel/family/FamilyReads.jsx @@ -296,7 +296,13 @@ class FamilyReads extends React.PureComponent { } updateReads = (familyGuid, locus, sampleTypes, tissueType) => { - this.setState({ openFamily: familyGuid, sampleTypes, locus, rnaReferences: TISSUE_REFERENCES_LOOKUP[tissueType] }) + const project = familyGuid && this.getProjectForFamily(familyGuid) + this.setState({ + openFamily: familyGuid, + sampleTypes, + locus, + rnaReferences: project?.genomeVersion === GENOME_VERSION_38 ? TISSUE_REFERENCES_LOOKUP[tissueType] : [], + }) } getProjectForFamily = (familyGuid) => { @@ -414,6 +420,7 @@ class FamilyReads extends React.PureComponent { const dnaTrackOptions = DNA_TRACK_TYPE_OPTIONS.filter(({ value }) => igvSampleIndividuals[value]) const rnaTrackOptions = RNA_TRACK_TYPE_OPTIONS.filter(({ value }) => igvSampleIndividuals[value]) const project = openFamily && this.getProjectForFamily(openFamily) + const isGrch38 = project?.genomeVersion === GENOME_VERSION_38 const geneLocus = project && variant && getGeneLocus(variant, genesById, project) const locusOptions = [ { text: 'Variant', value: geneLocus && getVariantLocus(variant) }, @@ -463,15 +470,17 @@ class FamilyReads extends React.PureComponent {
{sampleTypes.some(sampleType => RNA_TRACK_TYPE_LOOKUP.has(sampleType)) && (
- Reference Tracks - {this.gtexSelector('Normalized', NORM_GTEX_TRACK_OPTIONS)} - {this.gtexSelector('Aggregate', AGG_GTEX_TRACK_OPTIONS)} - + {isGrch38 && Reference Tracks} + {isGrch38 && this.gtexSelector('Normalized', NORM_GTEX_TRACK_OPTIONS)} + {isGrch38 && this.gtexSelector('Aggregate', AGG_GTEX_TRACK_OPTIONS)} + {isGrch38 && ( + + )} Junction Filters - {`Based on ${citation.name} (PMID: `} - {citation.pmid} + {`Based on ${citation.name} (${citation.pmid ? 'PMID: ' : ''}`} + + {citation.pmid || citation.linkText} + ) )} diff --git a/vlm/conftest.py b/vlm/conftest.py new file mode 100644 index 0000000000..449e667e14 --- /dev/null +++ b/vlm/conftest.py @@ -0,0 +1,36 @@ +import pytest + +@pytest.fixture +def _django_db_marker(_django_db_marker, db): + pass + +@pytest.fixture(scope='session') +def django_db_setup(request, django_db_blocker,django_db_keepdb): + from django.core.management import call_command + from django.test.utils import setup_databases, teardown_databases + from clickhouse_search.models.gt_stats_models import ProjectsToGtStatsGRCh37SnvIndel, ProjectsToGtStatsSnvIndel, \ + GtStatsDictGRCh37SnvIndel, GtStatsDictSnvIndel + + with django_db_blocker.unblock(): + db_cfg = setup_databases( + verbosity=request.config.option.verbose, + interactive=False, + aliases=['default', 'reference_data', 'clickhouse_write'], + keepdb=django_db_keepdb, + ) + call_command('loaddata', 'clickhouse_search', '--database=clickhouse_write') + ProjectsToGtStatsGRCh37SnvIndel.refresh() + ProjectsToGtStatsSnvIndel.refresh() + GtStatsDictGRCh37SnvIndel.reload() + GtStatsDictSnvIndel.reload() + + yield + + if not django_db_keepdb: + with django_db_blocker.unblock(): + try: + teardown_databases(db_cfg, verbosity=request.config.option.verbose) + except Exception as exc: # noqa: BLE001 + request.node.warn( + pytest.PytestWarning(f"Error when trying to teardown test databases: {exc!r}") + ) diff --git a/vlm/deploy/Dockerfile b/vlm/deploy/Dockerfile index 33a0fce9e5..51901775cc 100644 --- a/vlm/deploy/Dockerfile +++ b/vlm/deploy/Dockerfile @@ -2,13 +2,13 @@ FROM python:3.11-slim-bullseye LABEL maintainer="Broad TGG" -RUN pip install --no-cache-dir -r vlm/requirements.txt - WORKDIR /vlm # Application Code COPY vlm/ . +RUN pip install --no-cache-dir -r requirements.txt + WORKDIR / EXPOSE 7000 CMD ["python3", "-m", "vlm"] diff --git a/vlm/setup_clickhouse_test_data.py b/vlm/setup_clickhouse_test_data.py deleted file mode 100644 index a50334040c..0000000000 --- a/vlm/setup_clickhouse_test_data.py +++ /dev/null @@ -1,31 +0,0 @@ -import clickhouse_connect -import sys - -def setup_clickhouse_test_data(host, port, username, password): - client = clickhouse_connect.get_client(host=host, port=port, username=username, password=password) - client.command('CREATE DATABASE test_seqr') - - client.command('CREATE TABLE test_seqr.`GRCh37/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId') - client.insert('GRCh37/SNV_INDEL/key_lookup', data=[['7-143270172-A-G', 1], ['1-39190091-T-G', 2]], database='test_seqr') - client.command( - 'CREATE DICTIONARY test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` (key UInt32, ac_wes UInt32, ac_wgs UInt32, hom_wes UInt32, hom_wgs UInt32) PRIMARY KEY key SOURCE(CLICKHOUSE(USER %s PASSWORD %s QUERY "SELECT * FROM VALUES ((1, 4104, 607, 1276, 232), (2, 7, 2, 2, 1))")) LIFETIME(0) LAYOUT(FLAT(MAX_ARRAY_SIZE 500000000))', - parameters=(username, password), - ) - - client.command('CREATE TABLE test_seqr.`GRCh38/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId') - client.insert('GRCh38/SNV_INDEL/key_lookup', data=[['1-38724419-T-G', 1]], database='test_seqr') - client.command( - 'CREATE DICTIONARY test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` (key UInt32, ac_wes UInt32, ac_wgs UInt32, hom_wes UInt32, hom_wgs UInt32) PRIMARY KEY key SOURCE(CLICKHOUSE(USER %s PASSWORD %s QUERY "SELECT * FROM VALUES ((1, 18, 10, 3, 1))")) LIFETIME(0) LAYOUT(FLAT(MAX_ARRAY_SIZE 500000000))', - parameters=(username, password), - ) - - client.command("CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'") - client.command('GRANT SELECT ON test_seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user') - client.command('GRANT SELECT ON test_seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user') - client.command('GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user') - client.command('GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user') - - -if __name__ == '__main__': - args = sys.argv[1:] - setup_clickhouse_test_data(*args) diff --git a/vlm/test_vlm.py b/vlm/test_vlm.py index 02134c5e70..c048c0898e 100644 --- a/vlm/test_vlm.py +++ b/vlm/test_vlm.py @@ -68,7 +68,7 @@ async def test_match(self, mocked_responses): }, 'responseSummary': { 'exists': True, - 'total': 30, + 'total': 7, }, 'response': { 'resultSets': [ @@ -76,14 +76,14 @@ async def test_match(self, mocked_responses): 'exists': True, 'id': 'TestVLM Homozygous', 'results': [], - 'resultsCount': 7, + 'resultsCount': 3, 'setType': 'genomicVariant' }, { 'exists': True, 'id': 'TestVLM Heterozygous', 'results': [], - 'resultsCount': 23, + 'resultsCount': 4, 'setType': 'genomicVariant' }, { @@ -140,7 +140,7 @@ async def test_match(self, mocked_responses): }, 'responseSummary': { 'exists': True, - 'total': 3203, + 'total': 1, }, 'response': { 'resultSets': [ @@ -148,14 +148,14 @@ async def test_match(self, mocked_responses): 'exists': True, 'id': 'TestVLM Homozygous', 'results': [], - 'resultsCount': 1508, + 'resultsCount': 1, 'setType': 'genomicVariant' }, { - 'exists': True, + 'exists': False, 'id': 'TestVLM Heterozygous', 'results': [], - 'resultsCount': 1695, + 'resultsCount': 0, 'setType': 'genomicVariant' }, {