From 9623dde6c3021e727d84292fa6df4f85675c8e05 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 6 Apr 2026 13:43:29 -0400 Subject: [PATCH 001/119] create datasets --- seqr/migrations/0087_dataset.py | 114 ++++++++++++++++++++++++++++++++ seqr/models.py | 38 ++++++++++- 2 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 seqr/migrations/0087_dataset.py diff --git a/seqr/migrations/0087_dataset.py b/seqr/migrations/0087_dataset.py new file mode 100644 index 0000000000..17d413ebc9 --- /dev/null +++ b/seqr/migrations/0087_dataset.py @@ -0,0 +1,114 @@ +# Generated by Django 4.2.24 on 2026-04-06 16:39 + +from django.conf import settings +from django.contrib.postgres.aggregates import ArrayAgg +from django.db import migrations, models +from django.db.models.functions import Trim, TruncDate +import django.db.models.deletion +import django.utils.timezone + + +def set_individual_id(apps, schema_editor): + Sample = apps.get_model('seqr', 'Sample') + Individual = apps.get_model('seqr', 'Individual') + + db_alias = schema_editor.connection.alias + + mismatch_samples = Sample.objects.using(db_alias).filter(is_active=True).exclude(sample_id=models.F('individual__individual_id')) + individual_ids = [] + for sample in mismatch_samples: + individual = sample.individual + individual.display_name = individual.individual_id + # Since some of these are sample swamps, set a padded id to avoid conflicts and then trim whitespace for clean up + individual.individual_id = f' {sample.sample_id} ' + individual.save() + individual_ids.append(individual.id) + + updated = Individual.objects.using(db_alias).filter(id__in=individual_ids).update(individual_id=Trim('individual_id')) + if updated: + print(f'Updated individual_id for {updated} individuals to match their active sample_id') + + +def populate_datasets(apps, schema_editor): + Dataset = apps.get_model('seqr', 'Dataset') + Sample = apps.get_model('seqr', 'Sample') + db_alias = schema_editor.connection.alias + + dataset_aggs = Sample.objects.using(db_alias).values( + 'sample_type', 'dataset_type', project=models.F('individual__family__project'), group_loaded_date=TruncDate('loaded_date'), + ).annotate( + guid=models.Max('guid'), + loaded_date=models.Max('loaded_date'), + created_date=models.Max('created_date'), + last_modified_date=models.Max('last_modified_date'), + created_by_id=models.Min('created_by_id'), + data_sources=ArrayAgg('data_source', distinct=True, filter=models.Q(data_source__isnull=False)), + es_indices=ArrayAgg('elasticsearch_index', distinct=True, filter=models.Q(elasticsearch_index__isnull=False)), + active_individuals=ArrayAgg('individual__id', distinct=True, filter=models.Q(is_active=True)), + inactive_individuals=ArrayAgg('individual__id', distinct=True, filter=models.Q(is_active=False)), + ) + + datasets = [] + active_individuals = [] + inactive_individuals = [] + for dataset_agg in dataset_aggs: + del dataset_agg['project'] + del dataset_agg['group_loaded_date'] + active_individuals.append(dataset_agg.pop('active_individuals')) + inactive_individuals.append(dataset_agg.pop('inactive_individuals')) + data_sources = dataset_agg.pop('data_sources') + es_indices = dataset_agg.pop('es_indices') + datasets.append(Dataset(**dataset_agg, data_source=','.join(data_sources or es_indices))) + + created = Dataset.objects.using(db_alias).bulk_create(datasets) + if created: + print(f'Created {len(created)} datasets.') + + ActiveRelation = Dataset.active_individuals.through + InactiveRelation = Dataset.inactive_individuals.through + + active_relations = [] + for dataset, individual_ids in zip(created, active_individuals): + active_relations.extend(ActiveRelation(dataset_id=dataset.id, individual_id=ind_id) for ind_id in individual_ids) + ActiveRelation.objects.using(db_alias).bulk_create(active_relations) + if active_relations: + print(f'Created {len(active_relations)} active samples relations') + + inactive_relations = [] + for dataset, individual_ids in zip(created, inactive_individuals): + inactive_relations.extend(InactiveRelation(dataset_id=dataset.id, individual_id=ind_id) for ind_id in individual_ids) + InactiveRelation.objects.using(db_alias).bulk_create(inactive_relations) + if inactive_relations: + print(f'Created {len(inactive_relations)} inactive samples relations') + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('seqr', '0086_alter_rnasample_tissue_type'), + ] + + operations = [ + migrations.CreateModel( + name='Dataset', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('guid', models.CharField(db_index=True, max_length=30, unique=True)), + ('created_date', models.DateTimeField(db_index=True, default=django.utils.timezone.now)), + ('last_modified_date', models.DateTimeField(blank=True, db_index=True, null=True)), + ('sample_type', models.CharField(choices=[('WES', 'Exome'), ('WGS', 'Whole Genome')], max_length=10)), + ('dataset_type', models.CharField(choices=[('SNV_INDEL', 'Variant Calls'), ('SV', 'SV Calls'), ('MITO', 'Mitochondria calls')], max_length=13)), + ('data_source', models.TextField(null=True)), + ('loaded_date', models.DateTimeField()), + ('active_individuals', models.ManyToManyField(related_name='active_datasets', to='seqr.individual')), + ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)), + ('inactive_individuals', models.ManyToManyField(related_name='inactive_datasets', to='seqr.individual')), + ], + options={ + 'json_fields': ['guid', 'sample_type', 'dataset_type', 'loaded_date'], + }, + ), + migrations.RunPython(set_individual_id), + migrations.RunPython(populate_datasets), + ] diff --git a/seqr/models.py b/seqr/models.py index e7ab32f379..ff797b9cc5 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -616,7 +616,6 @@ class Individual(ModelWithGUID): sex = models.CharField(max_length=3, choices=SEX_CHOICES, default='U') affected = models.CharField(max_length=1, choices=AFFECTED_STATUS_CHOICES, default=AFFECTED_STATUS_UNKNOWN) - # TODO once sample and individual ids are fully decoupled no reason to maintain this field display_name = models.TextField(default="", blank=True) notes = models.TextField(blank=True, null=True) @@ -748,6 +747,43 @@ class Meta: ] +class Dataset(ModelWithGUID): + SAMPLE_TYPE_WES = 'WES' + SAMPLE_TYPE_WGS = 'WGS' + SAMPLE_TYPE_CHOICES = ( + (SAMPLE_TYPE_WES, 'Exome'), + (SAMPLE_TYPE_WGS, 'Whole Genome'), + ) + SAMPLE_TYPE_LOOKUP = dict(SAMPLE_TYPE_CHOICES) + + DATASET_TYPE_VARIANT_CALLS = 'SNV_INDEL' + DATASET_TYPE_SV_CALLS = 'SV' + DATASET_TYPE_MITO_CALLS = 'MITO' + DATASET_TYPE_CHOICES = ( + (DATASET_TYPE_VARIANT_CALLS, 'Variant Calls'), + (DATASET_TYPE_SV_CALLS, 'SV Calls'), + (DATASET_TYPE_MITO_CALLS, 'Mitochondria calls'), + ) + DATASET_TYPE_LOOKUP = dict(DATASET_TYPE_CHOICES) + + active_individuals = models.ManyToManyField('Individual', related_name='active_datasets') + inactive_individuals = models.ManyToManyField('Individual', related_name='inactive_datasets') + + sample_type = models.CharField(max_length=10, choices=SAMPLE_TYPE_CHOICES) + dataset_type = models.CharField(max_length=13, choices=DATASET_TYPE_CHOICES) + + data_source = models.TextField(null=True) + loaded_date = models.DateTimeField() + + def __unicode__(self): + return f'{self.dataset_type}_{self.sample_type}_{self.loaded_date}' + + GUID_PREFIX = 'D' + + class Meta: + json_fields = ['guid', 'sample_type', 'dataset_type', 'loaded_date'] + + class RnaSample(ModelWithGUID): DATA_TYPE_TPM = 'T' From 4c3b2593a5141baabebed7a0a480486cf6797bdb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 6 Apr 2026 13:53:08 -0400 Subject: [PATCH 002/119] sample model import cleanup pass --- .../commands/set_saved_variant_key.py | 18 ++++++------- clickhouse_search/models/gt_stats_models.py | 10 +++---- clickhouse_search/models/search_models.py | 26 +++++++++---------- seqr/admin.py | 4 +-- .../commands/update_individuals_sample_qc.py | 4 +-- seqr/utils/search/utils.py | 8 +++--- seqr/utils/vcf_utils.py | 8 +++--- seqr/views/apis/anvil_workspace_api.py | 6 ++--- seqr/views/apis/saved_variant_api.py | 4 +-- seqr/views/apis/summary_data_api.py | 4 +-- seqr/views/utils/orm_to_json_utils_tests.py | 10 ++----- 11 files changed, 48 insertions(+), 54 deletions(-) diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py index 2569dfc3a8..f743d44346 100644 --- a/clickhouse_search/management/commands/set_saved_variant_key.py +++ b/clickhouse_search/management/commands/set_saved_variant_key.py @@ -7,7 +7,7 @@ from clickhouse_search.search import get_clickhouse_key_lookup from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 -from seqr.models import SavedVariant, Sample +from seqr.models import SavedVariant, Dataset from seqr.utils.search.utils import parse_variant_id logger = logging.getLogger(__name__) @@ -23,7 +23,7 @@ def handle(self, *args, **options): saved_variant_json__populations__isnull=False, # Omit manual variants ).values_list('variant_id', flat=True).distinct() ids_by_dataset_type = { - Sample.DATASET_TYPE_VARIANT_CALLS: [], Sample.DATASET_TYPE_MITO_CALLS: [], Sample.DATASET_TYPE_SV_CALLS: [], + Dataset.DATASET_TYPE_VARIANT_CALLS: [], Dataset.DATASET_TYPE_MITO_CALLS: [], Dataset.DATASET_TYPE_SV_CALLS: [], } for variant_id in variant_ids: parsed_id = parse_variant_id(variant_id) @@ -32,23 +32,23 @@ def handle(self, *args, **options): parsed_id = parse_variant_id(variant_id[:-1]) if parsed_id: is_mito = parsed_id[0].replace('chr', '').startswith('M') - dataset_type = Sample.DATASET_TYPE_MITO_CALLS if is_mito else Sample.DATASET_TYPE_VARIANT_CALLS + dataset_type = Dataset.DATASET_TYPE_MITO_CALLS if is_mito else Dataset.DATASET_TYPE_VARIANT_CALLS else: - dataset_type = Sample.DATASET_TYPE_SV_CALLS + dataset_type = Dataset.DATASET_TYPE_SV_CALLS ids_by_dataset_type[dataset_type].append(variant_id) - no_key_mito = self._set_variant_keys(ids_by_dataset_type[Sample.DATASET_TYPE_MITO_CALLS], Sample.DATASET_TYPE_MITO_CALLS) + no_key_mito = self._set_variant_keys(ids_by_dataset_type[Dataset.DATASET_TYPE_MITO_CALLS], Dataset.DATASET_TYPE_MITO_CALLS) no_key_snv_indel = self._set_variant_keys( - ids_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] + list(no_key_mito), Sample.DATASET_TYPE_VARIANT_CALLS, + ids_by_dataset_type[Dataset.DATASET_TYPE_VARIANT_CALLS] + list(no_key_mito), Dataset.DATASET_TYPE_VARIANT_CALLS, ) if no_key_snv_indel: self._resolve_missing_variants(no_key_snv_indel, GENOME_VERSION_GRCh38) no_keys_svs = self._set_variant_keys( - ids_by_dataset_type[Sample.DATASET_TYPE_SV_CALLS], f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WGS}', + ids_by_dataset_type[Dataset.DATASET_TYPE_SV_CALLS], f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WGS}', ) - no_keys_svs = self._set_variant_keys(list(no_keys_svs), f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}') + no_keys_svs = self._set_variant_keys(list(no_keys_svs), f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WES}') if no_keys_svs: self._resolve_reloaded_svs(no_keys_svs) @@ -56,7 +56,7 @@ def handle(self, *args, **options): key__isnull=True, family__project__genome_version=GENOME_VERSION_GRCh37, ).values_list('variant_id', flat=True).distinct() - no_keys_37 = self._set_variant_keys(variant_ids_37, Sample.DATASET_TYPE_VARIANT_CALLS, genome_version=GENOME_VERSION_GRCh37) + no_keys_37 = self._set_variant_keys(variant_ids_37, Dataset.DATASET_TYPE_VARIANT_CALLS, genome_version=GENOME_VERSION_GRCh37) if no_keys_37: self._resolve_missing_variants(no_keys_37, GENOME_VERSION_GRCh37) diff --git a/clickhouse_search/models/gt_stats_models.py b/clickhouse_search/models/gt_stats_models.py index b6a12f2fb2..86ccb1e05a 100644 --- a/clickhouse_search/models/gt_stats_models.py +++ b/clickhouse_search/models/gt_stats_models.py @@ -5,7 +5,7 @@ from clickhouse_search.backend.table_models import RefreshableMaterializedView, RefreshableMaterializedViewMeta, \ IncrementalMaterializedView, Dictionary from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 -from seqr.models import Sample +from seqr.models import Dataset class BaseProjectGtStats(models.ClickhouseModel): @@ -308,10 +308,10 @@ class Meta(GtStatsDictMeta): layout = 'FLAT(MAX_ARRAY_SIZE 5000000)' PROJECT_GT_STATS_VIEW_CLASS_MAP = { - GENOME_VERSION_GRCh37: {Sample.DATASET_TYPE_VARIANT_CALLS: ProjectsToGtStatsGRCh37SnvIndel}, + GENOME_VERSION_GRCh37: {Dataset.DATASET_TYPE_VARIANT_CALLS: ProjectsToGtStatsGRCh37SnvIndel}, GENOME_VERSION_GRCh38: { - Sample.DATASET_TYPE_VARIANT_CALLS: ProjectsToGtStatsSnvIndel, - Sample.DATASET_TYPE_MITO_CALLS: ProjectsToGtStatsMito, - Sample.DATASET_TYPE_SV_CALLS: ProjectsToGtStatsSv, + Dataset.DATASET_TYPE_VARIANT_CALLS: ProjectsToGtStatsSnvIndel, + Dataset.DATASET_TYPE_MITO_CALLS: ProjectsToGtStatsMito, + Dataset.DATASET_TYPE_SV_CALLS: ProjectsToGtStatsSv, }, } diff --git a/clickhouse_search/models/search_models.py b/clickhouse_search/models/search_models.py index bcd47b2dcc..a42a0a5adb 100644 --- a/clickhouse_search/models/search_models.py +++ b/clickhouse_search/models/search_models.py @@ -10,7 +10,7 @@ from clickhouse_search.models.reference_data_models import GnomadNonCodingConstraintDict, BaseSpliceAi, \ ScreenDict from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 -from seqr.models import Sample +from seqr.models import Dataset from seqr.utils.xpos_utils import CHROMOSOME_CHOICES from settings import CLICKHOUSE_IN_MEMORY_DIR, CLICKHOUSE_DATA_DIR @@ -414,7 +414,7 @@ class Meta(BaseEntries.Meta): ) class EntriesSv(BaseEntries): - SAMPLE_TYPE = Sample.SAMPLE_TYPE_WGS + SAMPLE_TYPE = Dataset.SAMPLE_TYPE_WGS CALL_FIELDS = [ ('sampleId', models.StringField()), ('gt', models.Enum8Field(null=True, blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')])), @@ -436,7 +436,7 @@ class Meta(BaseEntries.Meta): db_table = 'GRCh38/SV/entries' class EntriesGcnv(BaseEntries): - SAMPLE_TYPE = Sample.SAMPLE_TYPE_WES + SAMPLE_TYPE = Dataset.SAMPLE_TYPE_WES CALL_FIELDS = [ ('sampleId', models.StringField()), ('gt', models.Enum8Field(null=True, blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')])), @@ -621,21 +621,21 @@ class Meta: ENTRY_CLASS_MAP = { - GENOME_VERSION_GRCh37: {Sample.DATASET_TYPE_VARIANT_CALLS: EntriesGRCh37SnvIndel}, + GENOME_VERSION_GRCh37: {Dataset.DATASET_TYPE_VARIANT_CALLS: EntriesGRCh37SnvIndel}, GENOME_VERSION_GRCh38: { - Sample.DATASET_TYPE_VARIANT_CALLS: EntriesSnvIndel, - Sample.DATASET_TYPE_MITO_CALLS: EntriesMito, - f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WGS}': EntriesSv, - f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}': EntriesGcnv, + Dataset.DATASET_TYPE_VARIANT_CALLS: EntriesSnvIndel, + Dataset.DATASET_TYPE_MITO_CALLS: EntriesMito, + f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WGS}': EntriesSv, + f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WES}': EntriesGcnv, }, } VARIANTS_CLASS_MAP = { - GENOME_VERSION_GRCh37: {Sample.DATASET_TYPE_VARIANT_CALLS: VariantsGRCh37SnvIndel}, + GENOME_VERSION_GRCh37: {Dataset.DATASET_TYPE_VARIANT_CALLS: VariantsGRCh37SnvIndel}, GENOME_VERSION_GRCh38: { - Sample.DATASET_TYPE_VARIANT_CALLS: VariantsSnvIndel, - Sample.DATASET_TYPE_MITO_CALLS: VariantsMito, - f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WGS}': VariantsSv, - f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}': VariantsGcnv, + Dataset.DATASET_TYPE_VARIANT_CALLS: VariantsSnvIndel, + Dataset.DATASET_TYPE_MITO_CALLS: VariantsMito, + f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WGS}': VariantsSv, + f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WES}': VariantsGcnv, }, } VARIANT_DETAILS_CLASS_MAP = { diff --git a/seqr/admin.py b/seqr/admin.py index 2ac70cc8e6..09d595c0be 100644 --- a/seqr/admin.py +++ b/seqr/admin.py @@ -1,12 +1,12 @@ from copy import deepcopy from django.contrib import admin from matchmaker.models import MatchmakerSubmission, MatchmakerIncomingQuery, MatchmakerResult, MatchmakerContactNotes -from seqr.models import Project, Family, Individual, Sample, LocusList, LocusListGene, LocusListInterval, VariantNote, \ +from seqr.models import Project, Family, Individual, Dataset, LocusList, LocusListGene, LocusListInterval, VariantNote, \ VariantTag, VariantTagType, VariantFunctionalData, SavedVariant, GeneNote, AnalysisGroup, ProjectCategory, \ FamilyAnalysedBy, VariantSearch, VariantSearchResults, IgvSample, UserPolicy, WarningMessage, FamilyNote, DynamicAnalysisGroup for model_class in [ - Project, Family, Individual, Sample, IgvSample, LocusList, LocusListGene, LocusListInterval, VariantNote, VariantTag, + Project, Family, Individual, Dataset, IgvSample, LocusList, LocusListGene, LocusListInterval, VariantNote, VariantTag, VariantTagType, VariantFunctionalData, SavedVariant, GeneNote, AnalysisGroup, ProjectCategory, FamilyAnalysedBy, VariantSearch, VariantSearchResults, MatchmakerSubmission, MatchmakerIncomingQuery, MatchmakerResult, MatchmakerContactNotes, FamilyNote, DynamicAnalysisGroup, diff --git a/seqr/management/commands/update_individuals_sample_qc.py b/seqr/management/commands/update_individuals_sample_qc.py index fcb3e7fcf5..eb9e5d6889 100644 --- a/seqr/management/commands/update_individuals_sample_qc.py +++ b/seqr/management/commands/update_individuals_sample_qc.py @@ -4,7 +4,7 @@ from django.core.management.base import BaseCommand, CommandError from reference_data.models import GENOME_VERSION_LOOKUP, GENOME_VERSION_GRCh38 -from seqr.models import Sample +from seqr.models import Dataset from seqr.management.commands.check_for_new_samples_from_pipeline import update_individuals_sample_qc, get_pipeline_runs from seqr.utils.file_utils import file_iter @@ -13,7 +13,7 @@ class Command(BaseCommand): help = 'Ingest sample qc data for a particular pipeline run' def add_arguments(self, parser): - parser.add_argument('dataset_type', choices={Sample.DATASET_TYPE_VARIANT_CALLS}) + parser.add_argument('dataset_type', choices={Dataset.DATASET_TYPE_VARIANT_CALLS}) parser.add_argument('genome_version', choices={GENOME_VERSION_LOOKUP[GENOME_VERSION_GRCh38]}) parser.add_argument('run_version') diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 2b52f3fe96..cb49b1c500 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -7,7 +7,7 @@ from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ get_sorted_search_results, clickhouse_variant_lookup, InvalidSearchException from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 -from seqr.models import Sample, Project, VariantSearchResults +from seqr.models import Dataset, Project, VariantSearchResults from seqr.utils.logging_utils import SeqrLogger from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json from clickhouse_search.constants import XPOS_SORT_KEY, PRIORITIZED_GENE_SORT, RECESSIVE, COMPOUND_HET, \ @@ -196,9 +196,9 @@ def _get_clickhouse_exclude_keys(search_hash, user): def variant_dataset_type(variant): if not parse_variant_id(variant['variantId']): - sample_type = Sample.SAMPLE_TYPE_WGS if 'endChrom' in variant else Sample.SAMPLE_TYPE_WES - return f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' - return Sample.DATASET_TYPE_MITO_CALLS if 'mitomapPathogenic' in variant else Sample.DATASET_TYPE_VARIANT_CALLS + sample_type = Dataset.SAMPLE_TYPE_WGS if 'endChrom' in variant else Dataset.SAMPLE_TYPE_WES + return f'{Dataset.DATASET_TYPE_SV_CALLS}_{sample_type}' + return Dataset.DATASET_TYPE_MITO_CALLS if 'mitomapPathogenic' in variant else Dataset.DATASET_TYPE_VARIANT_CALLS def get_variant_query_gene_counts(search_model, user): diff --git a/seqr/utils/vcf_utils.py b/seqr/utils/vcf_utils.py index 892985a490..312df0064f 100644 --- a/seqr/utils/vcf_utils.py +++ b/seqr/utils/vcf_utils.py @@ -5,7 +5,7 @@ from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.file_utils import file_iter, does_file_exist, list_files -from seqr.models import Sample +from seqr.models import Dataset BLOCK_SIZE = 65536 @@ -20,12 +20,12 @@ } DATA_TYPE_FORMAT_FIELDS = { - Sample.DATASET_TYPE_SV_CALLS: BASE_EXPECTED_FORMAT_FIELDS, + Dataset.DATASET_TYPE_SV_CALLS: BASE_EXPECTED_FORMAT_FIELDS, } DATA_TYPE_FILE_EXTS = { - Sample.DATASET_TYPE_MITO_CALLS: ('.mt',), - Sample.DATASET_TYPE_SV_CALLS: ('.bed', '.bed.gz'), + Dataset.DATASET_TYPE_MITO_CALLS: ('.mt',), + Dataset.DATASET_TYPE_SV_CALLS: ('.bed', '.bed.gz'), } REQUIRED_HEADERS = ['#CHROM', 'POS', 'ID', 'REF', 'ALT', 'QUAL', 'FILTER', 'INFO', 'FORMAT'] diff --git a/seqr/views/apis/anvil_workspace_api.py b/seqr/views/apis/anvil_workspace_api.py index 5106343a75..09dbcc82f0 100644 --- a/seqr/views/apis/anvil_workspace_api.py +++ b/seqr/views/apis/anvil_workspace_api.py @@ -10,7 +10,7 @@ from django.shortcuts import redirect from reference_data.models import GENOME_VERSION_LOOKUP -from seqr.models import Project, Family, CAN_EDIT, Sample, IgvSample +from seqr.models import Project, Family, CAN_EDIT, Dataset, IgvSample from seqr.views.react_app import render_app_html from seqr.views.utils.airtable_utils import AirtableSession, ANVIL_REQUEST_TRACKING_TABLE from seqr.views.utils.json_to_orm_utils import create_model_from_json @@ -222,7 +222,7 @@ def add_workspace_data(request, project_guid): error = 'Field(s) "{}" are required'.format(', '.join(missing_fields)) return create_json_response({'error': error}, status=400, reason=error) - pedigree_records, loaded_individual_ids, sample_type = _parse_uploaded_pedigree(request_json, project=project, search_dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) + pedigree_records, loaded_individual_ids, sample_type = _parse_uploaded_pedigree(request_json, project=project, search_dataset_type=Dataset.DATASET_TYPE_VARIANT_CALLS) loading_families = {record[JsonConstants.FAMILY_ID_COLUMN] for record in pedigree_records} pending_families = Family.objects.filter( @@ -302,7 +302,7 @@ def _trigger_add_workspace_data(project, pedigree_records, user, data_path, samp f"{data_path} to seqr project <{_get_seqr_project_url(project)}|*{project.name}*> (guid: {project.guid})" ) trigger_success = trigger_data_loading( - [project], individual_ids, sample_type, Sample.DATASET_TYPE_VARIANT_CALLS, project.genome_version, data_path, user=user, success_message=success_message, + [project], individual_ids, sample_type, Dataset.DATASET_TYPE_VARIANT_CALLS, project.genome_version, data_path, user=user, success_message=success_message, success_slack_channel=SEQR_SLACK_ANVIL_DATA_LOADING_CHANNEL, error_message=f'ERROR triggering AnVIL loading for project {project.guid}', ) AirtableSession(user, base=AirtableSession.ANVIL_BASE).safe_create_records( diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py index 3a1b289226..fa8cc3557f 100644 --- a/seqr/views/apis/saved_variant_api.py +++ b/seqr/views/apis/saved_variant_api.py @@ -4,7 +4,7 @@ from django.db.models import Q from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\ - Family, GeneNote, Project, Sample + Family, GeneNote, Project, Dataset from seqr.utils.xpos_utils import get_xpos from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json @@ -83,7 +83,7 @@ def create_manual_saved_variant_handler(request, family_guid): variant_json['saved_variant_json'] = {**variant_json} variant_json.update({ 'key': None, - 'dataset_type': Sample.DATASET_TYPE_SV_CALLS if variant_json.get('svName') else Sample.DATASET_TYPE_VARIANT_CALLS, + 'dataset_type': Dataset.DATASET_TYPE_SV_CALLS if variant_json.get('svName') else Dataset.DATASET_TYPE_VARIANT_CALLS, }) model_json = parse_saved_variant_json(variant_json, family.id) diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 7aa3f63934..2584721f95 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -11,7 +11,7 @@ get_mme_metrics, get_hpo_terms_by_id from matchmaker.models import MatchmakerSubmission from reference_data.models import HumanPhenotypeOntology -from seqr.models import Project, Family, Individual, Sample, VariantTagType, SavedVariant, FamilyAnalysedBy +from seqr.models import Project, Family, Individual, Dataset, VariantTagType, SavedVariant, FamilyAnalysedBy from seqr.views.utils.airtable_utils import AirtableSession from seqr.views.utils.file_utils import load_uploaded_file from seqr.utils.communication_utils import safe_post_to_slack @@ -276,7 +276,7 @@ def _get_clickhouse_variants(families_by_id: dict[int, dict], family_variant_ids families_by_project[family['project__guid']].append(family['guid']) for project_guid, family_guids in families_by_project.items(): genotype_keys = get_clickhouse_genotypes( - project_guid, family_guids, genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variants_by_key.keys(), + project_guid, family_guids, genome_version, Dataset.DATASET_TYPE_VARIANT_CALLS, variants_by_key.keys(), additional_fields=['xpos'] ) for key, entry_data in genotype_keys.items(): diff --git a/seqr/views/utils/orm_to_json_utils_tests.py b/seqr/views/utils/orm_to_json_utils_tests.py index f5a68c23a0..8f344bb717 100644 --- a/seqr/views/utils/orm_to_json_utils_tests.py +++ b/seqr/views/utils/orm_to_json_utils_tests.py @@ -1,12 +1,12 @@ from django.contrib.auth.models import User import mock from copy import deepcopy -from seqr.models import Project, Sample, IgvSample, SavedVariant, VariantNote, LocusList, VariantSearch +from seqr.models import Project, IgvSample, SavedVariant, VariantNote, LocusList, VariantSearch from seqr.views.utils.orm_to_json_utils import get_json_for_user, _get_json_for_project, \ get_json_for_sample, get_json_for_variant_note, get_json_for_locus_list, \ get_json_for_saved_searches, get_json_for_saved_variants_with_tags, get_json_for_current_user from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, \ - PROJECT_FIELDS, SAMPLE_FIELDS, SAVED_VARIANT_FIELDS, \ + PROJECT_FIELDS, SAVED_VARIANT_FIELDS, \ FUNCTIONAL_FIELDS, SAVED_SEARCH_FIELDS, LOCUS_LIST_DETAIL_FIELDS, PA_LOCUS_LIST_FIELDS, IGV_SAMPLE_FIELDS, \ TAG_FIELDS, VARIANT_NOTE_FIELDS @@ -95,12 +95,6 @@ def test_json_for_project(self): self.assertSetEqual(set(json.keys()), PROJECT_FIELDS) - def test_json_for_sample(self): - sample = Sample.objects.first() - json = get_json_for_sample(sample) - - self.assertSetEqual(set(json.keys()), SAMPLE_FIELDS) - def test_json_for_igv_sample(self): sample = IgvSample.objects.first() json = get_json_for_sample(sample) From e7121a5053f86173a77ddf00181d5fa104c3af38 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 6 Apr 2026 14:04:44 -0400 Subject: [PATCH 003/119] update fixture data --- seqr/fixtures/1kg_project.json | 329 +++------------------------------ 1 file changed, 25 insertions(+), 304 deletions(-) diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index 9957c85223..f1216876a1 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -909,38 +909,33 @@ } }, { - "model": "seqr.sample", + "model": "seqr.dataset", "pk": 129, "fields": { "guid": "S000129_na19675", "created_date": "2017-02-05T06:42:55.397Z", "created_by": null, "last_modified_date": "2017-03-13T09:07:49.744Z", - - "individual": 1, + "active_individuals": [1, 4, 5, 6, 7, 9], + "inactive_individuals": [3], "sample_type": "WES", "dataset_type": "SNV_INDEL", - "sample_id": "NA19675", - "is_active": true, - "elasticsearch_index": "test_index", + "data_source": "test_index", "loaded_date": "2017-02-05T06:12:55.397Z" } }, { - "model": "seqr.sample", + "model": "seqr.dataset", "pk": 130, "fields": { "guid": "S000130_na19678", "created_date": "2017-02-05T06:42:55.397Z", "created_by": null, "last_modified_date": "2017-03-13T09:07:49.744Z", - - "individual": 2, + "active_individuals": [2], "sample_type": "WES", "dataset_type": "SNV_INDEL", - "sample_id": "NA19678", - "is_active": true, - "elasticsearch_index": "test_index_old", + "data_source": "test_index_old", "loaded_date": "2017-02-05T06:13:55.397Z" } }, @@ -960,365 +955,91 @@ } }, { - "model": "seqr.sample", - "pk": 131, - "fields": { - "guid": "S000131_na19679", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:49.800Z", - - "sample_id": "NA19679", - "sample_type": "WES", - "is_active": false, - "elasticsearch_index": "test_index", - "individual": 3, - "dataset_type": "SNV_INDEL", - "loaded_date": "2017-02-05T06:15:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 132, - "fields": { - "guid": "S000132_hg00731", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:49.937Z", - "elasticsearch_index": "test_index", - "sample_id": "HG00731", - "sample_type": "WES", - "is_active": true, - "individual": 4, - "dataset_type": "SNV_INDEL", - "loaded_date": "2017-02-05T06:16:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 133, - "fields": { - "guid": "S000133_hg00732", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:49.963Z", - "elasticsearch_index": "test_index", - "sample_id": "HG00732", - "sample_type": "WES", - "is_active": true, - "individual": 5, - "dataset_type": "SNV_INDEL", - "loaded_date": "2017-02-05T06:17:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 134, - "fields": { - "guid": "S000134_hg00733", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:49.990Z", - "elasticsearch_index": "test_index", - "sample_id": "HG00733", - "sample_type": "WES", - "is_active": true, - "individual": 6, - "dataset_type": "SNV_INDEL", - "loaded_date": "2017-02-05T06:18:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 135, - "fields": { - "guid": "S000135_na20870", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:50.022Z", - "elasticsearch_index": "test_index", - "sample_id": "NA20870", - "sample_type": "WES", - "is_active": true, - "individual": 7, - "dataset_type": "SNV_INDEL", - "loaded_date": "2017-02-05T06:19:55.397Z" - } -}, -{ - "model": "seqr.sample", + "model": "seqr.dataset", "pk": 136, "fields": { "guid": "S000136_na20872", "created_date": "2017-02-05T06:42:55.397Z", "created_by": null, "last_modified_date": "2017-03-13T09:07:50.052Z", - "sample_id": "NA20872", "sample_type": "WES", - "is_active": false, - "individual": 8, + "inactive_individuals": [8, 10, 11, 12, 14, 17], "dataset_type": "SNV_INDEL", - "elasticsearch_index": "1kg.vcf.gz", + "data_source": "1kg.vcf.gz", "loaded_date": "2017-02-05T06:20:55.397Z" } }, { - "model": "seqr.sample", - "pk": 137, - "fields": { - "guid": "S000137_na20874", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:50.079Z", - "elasticsearch_index": "test_index", - "sample_id": "NA20874", - "sample_type": "WES", - "is_active": true, - "individual": 9, - "dataset_type": "SNV_INDEL", - "loaded_date": "2017-02-05T06:21:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 138, - "fields": { - "guid": "S000138_na20875", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:50.111Z", - "sample_id": "NA20875", - "sample_type": "WES", - "is_active": false, - "individual": 10, - "dataset_type": "SNV_INDEL", - "elasticsearch_index": "1kg.vcf.gz", - "loaded_date": "2017-02-05T06:22:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 139, - "fields": { - "guid": "S000139_na20876", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:50.136Z", - - "sample_id": "NA20876", - "sample_type": "WES", - "is_active": false, - "individual": 11, - "dataset_type": "SNV_INDEL", - "elasticsearch_index": "1kg.vcf.gz", - "loaded_date": "2017-02-05T06:23:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 140, - "fields": { - "guid": "S000140_na20877", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:50.165Z", - - "sample_id": "NA19678", - "sample_type": "WES", - "is_active": false, - "individual": 12, - "dataset_type": "SNV_INDEL", - "elasticsearch_index": "1kg.vcf.gz", - "loaded_date": "2017-02-05T06:24:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 142, - "fields": { - "guid": "S000142_na20881", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:50.220Z", - - "sample_id": "NA20881", - "sample_type": "WES", - "is_active": false, - "individual": 14, - "dataset_type": "SNV_INDEL", - "elasticsearch_index": "1kg.vcf.gz", - "loaded_date": "2017-02-05T06:25:55.397Z" - } -}, -{ - "model": "seqr.sample", + "model": "seqr.dataset", "pk": 143, "fields": { "guid": "S000143_na20885", "created_date": "2020-02-05T06:42:55.397Z", "created_by": null, "last_modified_date": "2020-03-13T09:07:50.247Z", - "elasticsearch_index": "test_index_second", - "sample_id": "NA20885", + "data_source": "test_index_second", "sample_type": "WGS", - "is_active": true, - "individual": 15, + "active_individuals": [15], "dataset_type": "SNV_INDEL", "loaded_date": "2020-02-05T06:26:55.397Z" } }, { - "model": "seqr.sample", + "model": "seqr.dataset", "pk": 144, "fields": { "guid": "S000144_na20888", "created_date": "2017-02-05T06:42:55.397Z", "created_by": null, "last_modified_date": "2017-03-13T09:07:50.277Z", - - "sample_id": "NA20888", "sample_type": "WGS", - "is_active": false, - "individual": 16, + "inactive_individuals": [16], "dataset_type": "SNV_INDEL", - "elasticsearch_index": "1kg.vcf.gz", + "data_source": "1kg.vcf.gz", "loaded_date": "2017-02-05T06:27:55.397Z" } }, { - "model": "seqr.sample", - "pk": 154, - "fields": { - "guid": "S000154_na20889", - "created_date": "2017-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2017-03-13T09:07:50.277Z", - - "sample_id": "NA20889", - "sample_type": "WES", - "is_active": false, - "individual": 17, - "dataset_type": "SNV_INDEL", - "elasticsearch_index": "1kg.vcf.gz", - "data_source": "auto__2023-08-08", - "loaded_date": "2017-02-05T06:28:55.397Z" - } -}, -{ - "model": "seqr.sample", + "model": "seqr.dataset", "pk": 145, "fields": { "guid": "S000145_hg00731", "created_date": "2018-02-05T06:42:55.397Z", "created_by": null, "last_modified_date": "2018-03-13T09:07:49.937Z", - "elasticsearch_index": "test_index_sv", - "sample_id": "HG00731", + "data_source": "test_index_sv", "sample_type": "WES", - "is_active": true, - "individual": 4, + "active_individuals": [4, 5, 6], "dataset_type": "SV", "loaded_date": "2018-02-05T06:29:55.397Z" } }, { - "model": "seqr.sample", - "pk": 146, - "fields": { - "guid": "S000146_hg00732", - "created_date": "2018-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2018-03-13T09:07:49.963Z", - "elasticsearch_index": "test_index_sv", - "sample_id": "HG00732", - "sample_type": "WES", - "is_active": true, - "individual": 5, - "dataset_type": "SV", - "loaded_date": "2018-02-05T06:30:55.397Z" - } -}, -{ - "model": "seqr.sample", + "model": "seqr.dataset", "pk": 147, "fields": { "guid": "S000147_na21234", "created_date": "2018-02-05T06:42:55.397Z", "created_by": null, "last_modified_date": "2018-03-13T09:07:49.937Z", - "elasticsearch_index": "test_index_sv_wgs", - "sample_id": "NA21234", - "sample_type": "WGS", - "is_active": true, - "individual": 18, - "dataset_type": "SV", - "loaded_date": "2018-02-05T06:31:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 173, - "fields": { - "guid": "S000173_na21987", - "created_date": "2018-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2018-03-13T09:07:49.937Z", - "elasticsearch_index": "test_index_sv_wgs", - "sample_id": "NA21987", + "data_source": "test_index_sv_wgs", "sample_type": "WGS", - "is_active": true, - "individual": 19, + "active_individuals": [18, 19, 21], "dataset_type": "SV", "loaded_date": "2018-02-05T06:31:55.397Z" } }, { - "model": "seqr.sample", - "pk": 174, - "fields": { - "guid": "S000174_na21654", - "created_date": "2018-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2018-03-13T09:07:49.937Z", - "elasticsearch_index": "test_index_sv_wgs", - "sample_id": "NA21654", - "sample_type": "WGS", - "is_active": true, - "individual": 21, - "dataset_type": "SV", - "loaded_date": "2018-02-05T06:31:55.397Z" - } -}, -{ - "model": "seqr.sample", - "pk": 148, - "fields": { - "guid": "S000148_hg00733", - "created_date": "2018-02-05T06:42:55.397Z", - "created_by": null, - "last_modified_date": "2018-03-13T09:07:49.963Z", - "elasticsearch_index": "test_index_sv", - "sample_id": "HG00733", - "sample_type": "WES", - "is_active": true, - "individual": 6, - "dataset_type": "SV", - "loaded_date": "2018-02-05T06:32:55.397Z" - } -}, -{ - "model": "seqr.sample", + "model": "seqr.dataset", "pk": 149, "fields": { "guid": "S000149_hg00733", "created_date": "2022-02-05T06:42:55.397Z", "created_by": null, "last_modified_date": "2022-03-13T09:07:49.937Z", - "elasticsearch_index": "test_index_mito_wgs", - "sample_id": "HG00731", + "data_source": "test_index_mito_wgs", "sample_type": "WES", - "is_active": true, - "individual": 4, + "active_individuals": [4], "dataset_type": "MITO", "loaded_date": "2022-02-05T06:33:55.397Z" } From 26370323dda7527798355b8f28db4f7254aed0d8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 6 Apr 2026 14:16:13 -0400 Subject: [PATCH 004/119] sample model constant clean up --- clickhouse_search/search.py | 50 +++++++++---------- .../check_for_new_samples_from_pipeline.py | 10 ++-- .../reload_saved_variant_genotypes.py | 4 +- .../commands/tag_seqr_prioritized_variants.py | 10 ++-- seqr/models.py | 8 +-- seqr/utils/search/add_data_utils.py | 6 +-- seqr/views/apis/dashboard_api.py | 4 +- seqr/views/apis/data_manager_api.py | 19 ++++--- seqr/views/apis/family_api.py | 4 +- seqr/views/apis/report_api.py | 4 +- seqr/views/apis/variant_search_api.py | 4 +- seqr/views/utils/anvil_metadata_utils.py | 8 +-- seqr/views/utils/variant_utils.py | 4 +- 13 files changed, 67 insertions(+), 68 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index fd37ecb84b..561d602a5c 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -16,7 +16,7 @@ from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, \ ENTRY_CLASS_MAP, VARIANTS_CLASS_MAP, VARIANT_DETAILS_CLASS_MAP from reference_data.models import GeneInfo, GeneConstraint, Omim, GENOME_VERSION_LOOKUP -from seqr.models import Sample, PhenotypePrioritization, Family, Individual +from seqr.models import Sample, Dataset, PhenotypePrioritization, Family, Individual from seqr.utils.logging_utils import SeqrLogger from clickhouse_search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \ PRIORITIZED_GENE_SORT, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS, RECESSIVE, AFFECTED, MALE_SEXES, \ @@ -55,7 +55,7 @@ def get_clickhouse_variants(families, search, user, genome_version, sort=None, s except InvalidDatasetTypeException: continue - if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and search.get('no_access_project_genome_version'): + if dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS and search.get('no_access_project_genome_version'): logger.info('Looking up variants in projects with no user access', user) results += _get_search_results( entry_qs, variants_qs, **search, **parsed_filters, sample_data=None, skip_entry_fields=True, @@ -84,7 +84,7 @@ def get_clickhouse_variants(families, search, user, genome_version, sort=None, s if has_comp_het: comp_het_sample_data = sample_data - if has_x_chrom_comp_het and 'affected_male_family_guids' in sample_data and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: + if has_x_chrom_comp_het and 'affected_male_family_guids' in sample_data and dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS: comp_het_sample_data = _no_affected_male_families(sample_data, user) result_q = _get_data_type_comp_het_results_queryset( entry_qs, variants_qs, comp_het_sample_data, exclude_key_pairs=exclude_key_pairs.get(dataset_type), @@ -98,7 +98,7 @@ def get_clickhouse_variants(families, search, user, genome_version, sort=None, s results += dataset_results searched_dataset_types.add(dataset_type) - if has_comp_het and any(dt.startswith(Sample.DATASET_TYPE_SV_CALLS) for dt in VARIANTS_CLASS_MAP[genome_version]): + if has_comp_het and any(dt.startswith(Dataset.DATASET_TYPE_SV_CALLS) for dt in VARIANTS_CLASS_MAP[genome_version]): results += _get_multi_data_type_comp_het_results(genome_version, families, sample_data_by_dataset_type, user, exclude_key_pairs, searched_dataset_types, **search) if not searched_dataset_types: @@ -144,8 +144,8 @@ def _raise_dataset_type_errors(sample_data_errors, sample_data_by_dataset_type): if sample_data_errors: raise InvalidSearchException(next(iter(sample_data_errors))) no_data_type = next(data_type for data_type, data in sample_data_by_dataset_type.items() if not data) - if no_data_type.startswith(Sample.DATASET_TYPE_SV_CALLS): - no_data_type = Sample.DATASET_TYPE_SV_CALLS + if no_data_type.startswith(Dataset.DATASET_TYPE_SV_CALLS): + no_data_type = Dataset.DATASET_TYPE_SV_CALLS raise InvalidSearchException(f'Unable to search against dataset type "{no_data_type}"') @@ -187,29 +187,29 @@ def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_d } try: - snv_indel_entry_qs = ENTRY_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS].objects.filter_locus(**search_kwargs) - snv_indel_variants_qs = VARIANTS_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS].objects + snv_indel_entry_qs = ENTRY_CLASS_MAP[genome_version][Dataset.DATASET_TYPE_VARIANT_CALLS].objects.filter_locus(**search_kwargs) + snv_indel_variants_qs = VARIANTS_CLASS_MAP[genome_version][Dataset.DATASET_TYPE_VARIANT_CALLS].objects snv_indel_parsed_filters = snv_indel_variants_qs.get_parsed_annotations_filters(annotations=annotations, **search_kwargs) - sv_variants_cls = VARIANTS_CLASS_MAP[genome_version][f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}'] + sv_variants_cls = VARIANTS_CLASS_MAP[genome_version][f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WES}'] sv_parsed_filters = sv_variants_cls.objects.get_parsed_annotations_filters(annotations=annotations, **search_kwargs) except InvalidDatasetTypeException: return [] - if Sample.DATASET_TYPE_VARIANT_CALLS not in sample_data_by_dataset_type: - sample_data_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] = _get_sample_data( + if Dataset.DATASET_TYPE_VARIANT_CALLS not in sample_data_by_dataset_type: + sample_data_by_dataset_type[Dataset.DATASET_TYPE_VARIANT_CALLS] = _get_sample_data( all_families, - Sample.DATASET_TYPE_VARIANT_CALLS, + Dataset.DATASET_TYPE_VARIANT_CALLS, inheritance_mode=COMPOUND_HET, inheritance_filter=search_kwargs.get('inheritance_filter'), ) - snv_indel_sample_data = sample_data_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] + snv_indel_sample_data = sample_data_by_dataset_type[Dataset.DATASET_TYPE_VARIANT_CALLS] if not (snv_indel_sample_data or {}).get('num_families'): return [] snv_indel_families = set().union(*snv_indel_sample_data['sample_type_families'].values()) results = [] - for sample_type in [Sample.SAMPLE_TYPE_WES, Sample.SAMPLE_TYPE_WGS]: - sv_dataset_type = f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' + for sample_type in [Dataset.SAMPLE_TYPE_WES, Dataset.SAMPLE_TYPE_WGS]: + sv_dataset_type = f'{Dataset.DATASET_TYPE_SV_CALLS}_{sample_type}' if sv_dataset_type not in sample_data_by_dataset_type: sample_data_by_dataset_type[sv_dataset_type] = _get_sample_data( all_families, @@ -222,7 +222,7 @@ def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_d families = snv_indel_families.intersection(sv_families) if not families: continue - logger.info(f'Loading {Sample.DATASET_TYPE_VARIANT_CALLS}/{sv_dataset_type} data for {len(families)} families', user) + logger.info(f'Loading {Dataset.DATASET_TYPE_VARIANT_CALLS}/{sv_dataset_type} data for {len(families)} families', user) snv_indel_sample_type_families = { sample_type: set(familes).intersection(sv_families) @@ -257,7 +257,7 @@ def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_d result_q = _get_comp_het_results_queryset( snv_indel_variants_qs, snv_indel_q, sv_q, len(families), - exclude_key_pairs=exclude_key_pairs.get(f'{Sample.DATASET_TYPE_VARIANT_CALLS},{sv_dataset_type}'), + exclude_key_pairs=exclude_key_pairs.get(f'{Dataset.DATASET_TYPE_VARIANT_CALLS},{sv_dataset_type}'), ) dataset_results = _evaluate_results(result_q, is_comp_het=True) if not sv_sample_data['samples']: @@ -417,7 +417,7 @@ def get_sorted_search_results(results, sort, families): def format_clickhouse_export_results(results, genome_version): formatted_results = [variant for result in results for variant in (result if isinstance(result, list) else [result])] keys_with_no_details = {result['key'] for result in formatted_results if not 'transcripts' in result} - detail_qs = get_variant_details_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details) + detail_qs = get_variant_details_queryset(genome_version, Dataset.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details) details_by_key = { detail['key']: detail for detail in detail_qs.values( 'key', 'rsid', mainTranscript=F('transcripts__0'), variantId=F('variant_id'), @@ -448,7 +448,7 @@ def format_clickhouse_results(results, genome_version): } details_by_key = { detail['key']: detail for detail in - get_variant_details_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details).result_values() + get_variant_details_queryset(genome_version, Dataset.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details).result_values() } formatted_results = [] @@ -553,7 +553,7 @@ def _get_sample_metadata(samples, affected_family_only, annotate_affected_males) def _get_sample_data(families, dataset_type, annotate_affected_males=False, allow_no_samples=False, inheritance_mode=None, inheritance_filter=None): sample_type = None - if dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS): + if dataset_type.startswith(Dataset.DATASET_TYPE_SV_CALLS): dataset_type, sample_type = dataset_type.split('_') samples = _get_valid_samples(families, dataset_type, sample_type, allow_no_samples) if not samples: @@ -610,7 +610,7 @@ def _add_missing_multi_type_samples(samples, data): ).filter(samples__len=1) for agg in individual_samples: sample = agg['samples'][0] - missing_type = Sample.SAMPLE_TYPE_WES if sample['sample_type'] == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS + missing_type = Dataset.SAMPLE_TYPE_WES if sample['sample_type'] == Dataset.SAMPLE_TYPE_WGS else Dataset.SAMPLE_TYPE_WGS data['family_missing_type_samples'][agg['family_guid']][missing_type].append(sample['sample_id']) @@ -818,7 +818,7 @@ def clickhouse_variant_lookup(user, variant_id, parsed_variant_id, sample_type, ) except InvalidDatasetTypeException: continue - if dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS): + if dataset_type.startswith(Dataset.DATASET_TYPE_SV_CALLS): if not sample_type: raise InvalidSearchException('Sample type must be specified to look up a structural variant') elif not dataset_type.endswith(sample_type): @@ -856,7 +856,7 @@ def clickhouse_variant_lookup(user, variant_id, parsed_variant_id, sample_type, if variant.get('svType') in {'DEL', 'DUP'}: other_sample_type, other_entry_class = next( (dt, cls) for dt, cls in ENTRY_CLASS_MAP[genome_version].items() - if dt != data_type and dt.startswith(Sample.DATASET_TYPE_SV_CALLS) + if dt != data_type and dt.startswith(Dataset.DATASET_TYPE_SV_CALLS) ) other_variants_cls = VARIANTS_CLASS_MAP[genome_version][other_sample_type] @@ -911,7 +911,7 @@ def get_variants_queryset(genome_version, dataset_type, keys, variant_ids=None): def get_variant_details_queryset(genome_version, dataset_type, keys): - if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: + if dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS: return VARIANT_DETAILS_CLASS_MAP[genome_version].objects.filter(key__in=keys) return get_variants_queryset(genome_version, dataset_type, keys) @@ -940,7 +940,7 @@ def _main_transcript(selected_transcript_id, sorted_transcripts): def delete_clickhouse_project(project, dataset_type, sample_type=None): - if dataset_type == Sample.DATASET_TYPE_SV_CALLS and sample_type == Sample.SAMPLE_TYPE_WES: + if dataset_type == Dataset.DATASET_TYPE_SV_CALLS and sample_type == Dataset.SAMPLE_TYPE_WES: dataset_type = 'GCNV' table_base = f'{GENOME_VERSION_LOOKUP[project.genome_version]}/{dataset_type}' with connections['clickhouse_write'].cursor() as cursor: diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 3ce9618d97..518e5fa8c4 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -10,7 +10,7 @@ from clickhouse_search.search import get_clickhouse_genotypes from reference_data.models import GENOME_VERSION_LOOKUP -from seqr.models import Family, Sample, Project, Individual, SavedVariant +from seqr.models import Family, Sample, Dataset, Project, Individual, SavedVariant from seqr.utils.communication_utils import safe_post_to_slack, send_project_email from seqr.utils.file_utils import file_iter, list_files, is_google_bucket_file_path from seqr.utils.search.add_data_utils import notify_search_data_loaded, update_airtable_loading_tracking_status @@ -30,10 +30,10 @@ ERRORS_REPORTED_FILE_NAME = '_ERRORS_REPORTED' RUN_PATH_FIELDS = ['genome_version', 'dataset_type', 'run_version', 'file_name'] -DATASET_TYPE_MAP = {'GCNV': Sample.DATASET_TYPE_SV_CALLS} +DATASET_TYPE_MAP = {'GCNV': Dataset.DATASET_TYPE_SV_CALLS} CLICKHOUSE_DATASET_TYPE_MAP = { - 'GCNV': f'{Sample.DATASET_TYPE_SV_CALLS}_WES', - Sample.DATASET_TYPE_SV_CALLS: f'{Sample.DATASET_TYPE_SV_CALLS}_WGS', + 'GCNV': f'{Dataset.DATASET_TYPE_SV_CALLS}_WES', + Dataset.DATASET_TYPE_SV_CALLS: f'{Dataset.DATASET_TYPE_SV_CALLS}_WGS', } RELATEDNESS_CHECK_NAME = 'relatedness_check' @@ -271,7 +271,7 @@ def _report_loading_success(cls, dataset_type, sample_type, run_version, samples project, is_internal, dataset_type, sample_type, new_samples_by_project.get(project.id, []), num_samples=len(sample_ids), ) - if session and is_internal and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: + if session and is_internal and dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS: split_project_pdos[project.name] = cls._update_pdos(session, project.guid, sample_ids) except Exception as e: logger.error(f'Error reporting loading success for project {project.name} in {run_version}: {e}') diff --git a/seqr/management/commands/reload_saved_variant_genotypes.py b/seqr/management/commands/reload_saved_variant_genotypes.py index 3cd146ac40..86b5c20b14 100644 --- a/seqr/management/commands/reload_saved_variant_genotypes.py +++ b/seqr/management/commands/reload_saved_variant_genotypes.py @@ -1,7 +1,7 @@ import logging from django.core.management.base import BaseCommand from seqr.management.commands.check_for_new_samples_from_pipeline import update_project_saved_variant_genotypes -from seqr.models import Project, Family, Sample +from seqr.models import Project, Family, Sample, Dataset logger = logging.getLogger(__name__) @@ -22,7 +22,7 @@ def handle(self, *args, **options): if family_guid: samples = samples.filter(individual__family__guid=family_guid) dataset_types = { - f'{dataset_type}_{sample_type}' if dataset_type == Sample.DATASET_TYPE_SV_CALLS else dataset_type + f'{dataset_type}_{sample_type}' if dataset_type == Dataset.DATASET_TYPE_SV_CALLS else dataset_type for dataset_type, sample_type in samples.values_list('dataset_type', 'sample_type').distinct() } for dataset_type in sorted(dataset_types): diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index 71b4a1cd9b..52d775f33c 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -8,7 +8,7 @@ from clickhouse_search.search import get_clickhouse_variants, ENTRY_CLASS_MAP from panelapp.models import PaLocusListGene from reference_data.models import GENOME_VERSION_GRCh38 -from seqr.models import Project, Family, Individual, Sample, LocusList +from seqr.models import Project, Family, Individual, Sample, Dataset, LocusList from seqr.utils.communication_utils import send_project_notification from seqr.utils.gene_utils import get_genes from clickhouse_search.constants import ANY_AFFECTED, HOMOZYGOUS_RECESSIVE, X_LINKED_RECESSIVE_MALE_AFFECTED, DE_NOVO, COMPOUND_HET @@ -506,7 +506,7 @@ def handle(self, *args, **options): @classmethod def _run_dataset_type_searches(cls, dataset_type, searches, sample_qs, updates, search_counts, samples_by_dataset_type, family_guid_map, project, exclude_genes, gene_by_moi): - is_sv = dataset_type == Sample.DATASET_TYPE_SV_CALLS + is_sv = dataset_type == Dataset.DATASET_TYPE_SV_CALLS sample_qs = sample_qs.filter(dataset_type=dataset_type) if is_sv: sample_qs = sample_qs.exclude( @@ -596,12 +596,12 @@ def wrapped(new_variant_keys, family_variant_data): def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by_dataset_type, family_guid_map, project, genes): sv_dataset_type = next(dt for dt in samples_by_dataset_type.keys() if dt.startswith('SV')) sample_type = sv_dataset_type.split('_')[-1] - families = set(samples_by_dataset_type[sv_dataset_type].keys()).intersection(samples_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS].keys()) + families = set(samples_by_dataset_type[sv_dataset_type].keys()).intersection(samples_by_dataset_type[Dataset.DATASET_TYPE_VARIANT_CALLS].keys()) sv_samples_by_family = { guid: sample_data for guid, sample_data in samples_by_dataset_type[sv_dataset_type].items() if guid in families } snv_indel_samples_by_family = { - guid: sample_data for guid, sample_data in samples_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS].items() + guid: sample_data for guid, sample_data in samples_by_dataset_type[Dataset.DATASET_TYPE_VARIANT_CALLS].items() if guid in families } family_variant_data = defaultdict(lambda: {'matched_searches': set(), 'matched_comp_het_searches': set(), 'support_vars': set()}) @@ -615,7 +615,7 @@ def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by project, sample_type, snv_indel_samples_by_family, config_search.get('family_filter'), ) sample_data_by_dataset_type = { - Sample.DATASET_TYPE_VARIANT_CALLS: snv_indel_sample_data, sv_dataset_type: sv_sample_data, + Dataset.DATASET_TYPE_VARIANT_CALLS: snv_indel_sample_data, sv_dataset_type: sv_sample_data, } num_results = cls._execute_search( sample_data_by_dataset_type, search_name, family_variant_data, family_guid_map, diff --git a/seqr/models.py b/seqr/models.py index ff797b9cc5..83878af0e7 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -864,10 +864,10 @@ class Meta: class SavedVariant(ModelWithGUID): DATASET_TYPE_CHOICES = ( - (Sample.DATASET_TYPE_VARIANT_CALLS, 'Variant Calls'), - (Sample.DATASET_TYPE_MITO_CALLS, 'Mitochondria calls'), - (f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WGS}', 'SV WGS Calls'), - (f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}', 'gCNV Calls'), + (Dataset.DATASET_TYPE_VARIANT_CALLS, 'Variant Calls'), + (Dataset.DATASET_TYPE_MITO_CALLS, 'Mitochondria calls'), + (f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WGS}', 'SV WGS Calls'), + (f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WES}', 'gCNV Calls'), ) family = models.ForeignKey('Family', on_delete=models.CASCADE) diff --git a/seqr/utils/search/add_data_utils.py b/seqr/utils/search/add_data_utils.py index a7a5c1a2c9..a9de1385b9 100644 --- a/seqr/utils/search/add_data_utils.py +++ b/seqr/utils/search/add_data_utils.py @@ -5,7 +5,7 @@ import requests from reference_data.models import GeneInfo, GENOME_VERSION_LOOKUP -from seqr.models import Sample, Individual, Project +from seqr.models import Sample, Dataset, Individual, Project from seqr.utils.communication_utils import send_project_notification, safe_post_to_slack from seqr.utils.file_utils import does_file_exist from seqr.utils.logging_utils import SeqrLogger @@ -21,7 +21,7 @@ def basic_notify_search_data_loaded(project, dataset_type, sample_type, new_samples, email_template=None, is_internal=True): - msg_dataset_type = '' if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS else f' {dataset_type}' + msg_dataset_type = '' if dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS else f' {dataset_type}' num_new_samples = len(new_samples) sample_summary = f'{num_new_samples} new {sample_type}{msg_dataset_type} samples' @@ -142,7 +142,7 @@ def _enqueue_pipeline_request(name: str, variables: dict, user: User, raise_erro def _loading_dataset_type(sample_type: str, dataset_type: str): - return 'GCNV' if dataset_type == Sample.DATASET_TYPE_SV_CALLS and sample_type == Sample.SAMPLE_TYPE_WES \ + return 'GCNV' if dataset_type == Dataset.DATASET_TYPE_SV_CALLS and sample_type == Dataset.SAMPLE_TYPE_WES \ else dataset_type diff --git a/seqr/views/apis/dashboard_api.py b/seqr/views/apis/dashboard_api.py index 1c20f6216b..efd476700c 100644 --- a/seqr/views/apis/dashboard_api.py +++ b/seqr/views/apis/dashboard_api.py @@ -3,7 +3,7 @@ """ from django.db import models -from seqr.models import ProjectCategory, Sample, RnaSample, Family, Project +from seqr.models import ProjectCategory, Sample, Dataset, RnaSample, Family, Project from seqr.views.utils.individual_utils import check_project_individuals_deletable from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.orm_to_json_utils import get_json_for_projects @@ -60,7 +60,7 @@ def _get_projects_json(user): projects_by_guid[project_guid]['analysisStatusCounts'][agg['analysis_status']] = agg['count'] sample_type_status_counts = _sample_type_counts( - Sample.objects.filter(individual__family__project__in=projects, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS) + Sample.objects.filter(individual__family__project__in=projects, dataset_type=Dataset.DATASET_TYPE_VARIANT_CALLS) ) + _sample_type_counts( RnaSample.objects.filter(individual__family__project__in=projects).annotate(sample_type=models.Value('RNA')) ) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 6f0fc97ae9..fb3f8799b1 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -23,7 +23,7 @@ from seqr.views.utils.permissions_utils import data_manager_required, pm_or_data_manager_required, get_internal_projects from seqr.views.utils.terra_api_utils import anvil_enabled -from seqr.models import Sample, RnaSample, Individual, Project, PhenotypePrioritization +from seqr.models import Sample, Dataset, RnaSample, Individual, Project, PhenotypePrioritization from settings import LOADING_DATASETS_DIR, LUIGI_UI_SERVICE_HOSTNAME, LUIGI_UI_SERVICE_PORT @@ -219,8 +219,8 @@ def get_loaded_projects(request, genome_version, sample_type, dataset_type): except ValueError as e: return create_json_response({'error': str(e)}, status=400) projects = projects.filter(guid__in=project_samples.keys()) - if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: - exclude_sample_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS + if dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS: + exclude_sample_type = Dataset.SAMPLE_TYPE_WES if sample_type == Dataset.SAMPLE_TYPE_WGS else Dataset.SAMPLE_TYPE_WGS # Include projects with either the matched sample type OR with no loaded data projects = projects.exclude(family__individual__sample__sample_type=exclude_sample_type) else: @@ -240,10 +240,10 @@ def get_loaded_projects(request, genome_version, sample_type, dataset_type): AIRTABLE_CALLSET_FIELDS = { - (Sample.DATASET_TYPE_MITO_CALLS, Sample.SAMPLE_TYPE_WES): 'MITO_WES_CallsetPath', - (Sample.DATASET_TYPE_MITO_CALLS, Sample.SAMPLE_TYPE_WGS): 'MITO_WGS_CallsetPath', - (Sample.DATASET_TYPE_SV_CALLS, Sample.SAMPLE_TYPE_WES): 'gCNV_CallsetPath', - (Sample.DATASET_TYPE_SV_CALLS, Sample.SAMPLE_TYPE_WGS): 'SV_CallsetPath', + (Dataset.DATASET_TYPE_MITO_CALLS, Dataset.SAMPLE_TYPE_WES): 'MITO_WES_CallsetPath', + (Dataset.DATASET_TYPE_MITO_CALLS, Dataset.SAMPLE_TYPE_WGS): 'MITO_WGS_CallsetPath', + (Dataset.DATASET_TYPE_SV_CALLS, Dataset.SAMPLE_TYPE_WES): 'gCNV_CallsetPath', + (Dataset.DATASET_TYPE_SV_CALLS, Dataset.SAMPLE_TYPE_WGS): 'SV_CallsetPath', } @@ -271,7 +271,7 @@ def load_data(request): request_json = json.loads(request.body) vcf_samples = request_json['vcfSamples'] sample_type = request_json['sampleType'] - dataset_type = request_json.get('datasetType', Sample.DATASET_TYPE_VARIANT_CALLS) + dataset_type = request_json.get('datasetType', Dataset.DATASET_TYPE_VARIANT_CALLS) projects = [json.loads(project) for project in request_json['projects']] project_samples = {p['projectGuid']: p.get('sampleIds') for p in projects} @@ -403,7 +403,7 @@ def trigger_delete_project(request): samples = Sample.objects.filter(individual__family__project=project, dataset_type=dataset_type, is_active=True) sample_types = list( samples.values_list('sample_type', flat=True).distinct() - ) if dataset_type == Sample.DATASET_TYPE_SV_CALLS else [None] + ) if dataset_type == Dataset.DATASET_TYPE_SV_CALLS else [None] updated = Sample.bulk_update(user=request.user, update_json={'is_active': False}, queryset=samples) info = [f'Deactivated search for {len(updated)} individuals'] for sample_type in sample_types: @@ -416,7 +416,6 @@ def trigger_delete_family(request): request_json = json.loads(request.body) family_guid = request_json.pop('family') project = Project.objects.get(family__guid=family_guid) - samples = Sample.objects.filter(individual__family__guid=family_guid) info = trigger_delete_families_search(project, [family_guid], request.user) return create_json_response({'info': info}) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index d3254d5203..50c378ab81 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -21,7 +21,7 @@ get_json_for_matchmaker_submissions, get_json_for_analysis_groups, _get_json_for_families, get_json_for_queryset from seqr.views.utils.project_context_utils import add_families_context, families_discovery_tags, add_project_tag_types, \ MME_TAG_NAME -from seqr.models import Family, FamilyAnalysedBy, Individual, FamilyNote, Sample, VariantTag, AnalysisGroup, RnaSeqTpm, \ +from seqr.models import Family, FamilyAnalysedBy, Individual, FamilyNote, Sample, Dataset, VariantTag, AnalysisGroup, RnaSeqTpm, \ PhenotypePrioritization, Project, RnaSample from seqr.views.utils.permissions_utils import check_project_permissions, get_project_and_check_pm_permissions, \ login_and_policies_required, user_is_analyst, has_case_review_permissions, external_anvil_project_can_edit, \ @@ -63,7 +63,7 @@ def family_page_data(request, family_guid): gene_ids = {gene_id for variant in discovery_variants for gene_id in variant['gene_ids']} discovery_variant_intervals = [dict(zip( ['chrom', 'start', 'end_chrom', 'end', 'svType', 'hasSvType'], - [*get_chrom_pos(v['xpos']), *get_chrom_pos(v['xpos_end']), v['svType'], (v.get('dataset_type') or '').startswith(Sample.DATASET_TYPE_SV_CALLS)] + [*get_chrom_pos(v['xpos']), *get_chrom_pos(v['xpos_end']), v['svType'], (v.get('dataset_type') or '').startswith(Dataset.DATASET_TYPE_SV_CALLS)] )) for v in discovery_variants] omims = Omim.objects.filter( get_omim_intervals_query(discovery_variant_intervals) | Q(gene__gene_id__in=gene_ids) diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index 7f1dc425f3..e6f372b76b 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -24,7 +24,7 @@ from seqr.views.utils.terra_api_utils import anvil_enabled from seqr.views.utils.variant_utils import DISCOVERY_CATEGORY -from seqr.models import Project, Family, FamilyAnalysedBy, Sample, RnaSample, Individual +from seqr.models import Project, Family, FamilyAnalysedBy, Sample, Dataset, RnaSample, Individual from settings import GREGOR_DATA_MODEL_URL @@ -1065,6 +1065,6 @@ def _get_clickhouse_metadata(dataset_type, genome_version, keys, include_clinvar if include_clinvar: fields.append('clinvar') qs = qs.join_clinvar() - if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: + if dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS: fields.append('caid') return {(dataset_type, genome_version, v.pop('key')): v for v in qs.values(*fields)} diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index e0c62e77d8..1728375f58 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -13,7 +13,7 @@ import re from reference_data.models import GENOME_VERSION_GRCh38 -from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Sample +from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Sample, Dataset from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, \ variant_lookup, parse_variant_id, export_variants from seqr.utils.search.utils import InvalidSearchException @@ -104,7 +104,7 @@ def _get_or_create_results_model(search_hash, search_context, user): families = families.exclude(analysis_status__in=Family.SOLVED_ANALYSIS_STATUSES) if search_context.get('trioFamiliesOnly'): families = families.annotate(search_sample_count=Count('individual__sample__id', filter=Q( - individual__sample__is_active=True, individual__sample__dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS, + individual__sample__is_active=True, individual__sample__dataset_type=Dataset.DATASET_TYPE_VARIANT_CALLS, ))).filter( search_sample_count__gte=3, individual__mother__isnull=False, individual__father__isnull=False, ).distinct() diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 5d9d860ab0..eefe6863bb 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -10,7 +10,7 @@ from matchmaker.models import MatchmakerSubmission from reference_data.models import HumanPhenotypeOntology, Omim, GENOME_VERSION_LOOKUP -from seqr.models import Project, Family, Individual, Sample, SavedVariant, VariantTagType +from seqr.models import Project, Family, Individual, Sample, Dataset, SavedVariant, VariantTagType from seqr.views.utils.airtable_utils import AirtableSession from seqr.utils.gene_utils import get_genes from seqr.utils.middleware import ErrorsWarningsException @@ -96,8 +96,8 @@ } METHOD_MAP = { - Sample.SAMPLE_TYPE_WES: 'SR-ES', - Sample.SAMPLE_TYPE_WGS: 'SR-GS', + Dataset.SAMPLE_TYPE_WES: 'SR-ES', + Dataset.SAMPLE_TYPE_WGS: 'SR-GS', } FAMILY_INDIVIDUAL_FIELDS = ['family_id', 'internal_project_id', 'phenotype_description', 'pmid_id', 'solve_status'] @@ -510,7 +510,7 @@ def _get_genetic_findings_rows(rows: list[dict], individual: Individual, family_ ]) if individual_data_types is not None: parsed_row['method_of_discovery'] = '|'.join([ - METHOD_MAP.get(data_type) for data_type in individual_data_types if data_type in Sample.SAMPLE_TYPE_LOOKUP + METHOD_MAP.get(data_type) for data_type in individual_data_types if data_type in Dataset.SAMPLE_TYPE_LOOKUP ]) if sample is not None: parsed_row['sample_id'] = sample.sample_id diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 7a3aed8dc1..37ce9975f2 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -12,7 +12,7 @@ from matchmaker.models import MatchmakerSubmissionGenes, MatchmakerSubmission from reference_data.models import TranscriptInfo, Omim, GENOME_VERSION_GRCh38 from seqr.models import SavedVariant, VariantSearchResults, Family, LocusList, LocusListInterval, LocusListGene, \ - RnaSeqTpm, PhenotypePrioritization, Project, Sample, RnaSample, VariantTag, VariantTagType + RnaSeqTpm, PhenotypePrioritization, Project, Sample, Dataset, RnaSample, VariantTag, VariantTagType from seqr.utils.search.utils import variant_dataset_type from seqr.utils.gene_utils import get_genes_for_variants from seqr.views.utils.json_to_orm_utils import create_model_from_json @@ -171,7 +171,7 @@ def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], comp_het_ def get_saved_variant_annotations(variant_keys: abc.Iterable[tuple[int, str]], genome_version: str, primary_id_field: str = 'variant_id', group_by_field=None, dataset_type: str = None) -> dict[str, dict]: - dataset_type = dataset_type or Sample.DATASET_TYPE_VARIANT_CALLS + dataset_type = dataset_type or Dataset.DATASET_TYPE_VARIANT_CALLS variant_ids = {variant_id for _, variant_id in variant_keys} keys = None if primary_id_field == 'key': From 02cd4d329848fcbf169966115295276b23b8129e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 6 Apr 2026 14:51:25 -0400 Subject: [PATCH 005/119] fix seqr stats --- seqr/views/apis/report_api.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index e6f372b76b..f300067758 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -1,7 +1,7 @@ from collections import defaultdict from datetime import datetime, timedelta -from django.db.models import Count, Q, F, Value +from django.db.models import Count, Q, F, Sum, Value from django.contrib.postgres.aggregates import ArrayAgg import json import re @@ -24,7 +24,7 @@ from seqr.views.utils.terra_api_utils import anvil_enabled from seqr.views.utils.variant_utils import DISCOVERY_CATEGORY -from seqr.models import Project, Family, FamilyAnalysedBy, Sample, Dataset, RnaSample, Individual +from seqr.models import Project, Family, FamilyAnalysedBy, Dataset, RnaSample, Individual from settings import GREGOR_DATA_MODEL_URL @@ -60,10 +60,13 @@ def seqr_stats(request): grouped_sample_counts = defaultdict(dict) for project_key, projects in project_models.items(): - samples_counts = _get_sample_counts(Sample.objects.filter(individual__family__project__in=projects)) + samples_counts = _get_sample_counts( + Dataset.objects.filter(active_individuals__family__project__in=projects), + count=Count('active_individuals'), + ) samples_counts.update(_get_sample_counts( - RnaSample.objects.filter(individual__family__project__in=projects).annotate(sample_type=Value('RNA')), - data_type_key='data_type') + RnaSample.objects.filter(individual__family__project__in=projects, is_active=True).annotate(sample_type=Value('RNA')), + data_type_key='data_type', count=Count('*')) ) for k, v in samples_counts.items(): grouped_sample_counts[k][project_key] = v @@ -80,8 +83,8 @@ def seqr_stats(request): }) -def _get_sample_counts(sample_q, data_type_key='dataset_type'): - samples_agg = sample_q.filter(is_active=True).values('sample_type', data_type_key).annotate(count=Count('*')) +def _get_sample_counts(sample_q, count=None, data_type_key='dataset_type'): + samples_agg = sample_q.values('sample_type', data_type_key).annotate(count=count) return { f'{sample_agg["sample_type"]}__{sample_agg[data_type_key]}': sample_agg['count'] for sample_agg in samples_agg } @@ -513,20 +516,18 @@ def _process_participant_row(participant, phenotype_rows, missing_participant_id def _get_individual_data_types(projects): - sample_types = Sample.objects.filter(individual__family__project__in=projects).values_list('individual_id', 'sample_type') - individual_data_types = defaultdict(set) - for individual_db_id, sample_type in sample_types: - individual_data_types[individual_db_id].add(sample_type) - for individual_db_id in RnaSample.objects.filter(individual__family__project__in=projects).values_list('individual_id', flat=True): - individual_data_types[individual_db_id].add('RNA') - individuals = Individual.objects.filter(id__in=individual_data_types).prefetch_related( + rna_individuals = set(RnaSample.objects.filter(individual__family__project__in=projects).values_list('individual_id', flat=True)) + individuals = Individual.objects.filter(family__project__in=projects).annotate( + sample_types=ArrayAgg('active_datasets__sample_type', distinct=True), + ).prefetch_related( 'family__project', 'mother', 'father') grouped_data_type_individuals = defaultdict(dict) for i in individuals: participant_id = _format_gregor_id(i.individual_id) - grouped_data_type_individuals[participant_id].update( - {data_type: i for data_type in individual_data_types[i.id]}) + data_types = set(i.sample_types) | ({'RNA'} if i.id in rna_individuals else set()) + if data_types: + grouped_data_type_individuals[participant_id].update({data_type: i for data_type in data_types}) return grouped_data_type_individuals From bf91082b1fecf9834ec66dab7b384191eea25231 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 6 Apr 2026 15:36:00 -0400 Subject: [PATCH 006/119] fix anvil meatdat autils --- seqr/views/apis/report_api.py | 19 +++--- seqr/views/utils/anvil_metadata_utils.py | 81 +++++++++++++----------- 2 files changed, 53 insertions(+), 47 deletions(-) diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index f300067758..1bbef12668 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -179,8 +179,8 @@ def _add_row(row, family_id, row_type): 'congenital_status': Individual.ONSET_AGE_LOOKUP[individual.onset_age] if individual.onset_age else 'Unknown', **anvil_export_airtable_fields(airtable_metadata, has_dbgap_submission), }, - get_additional_sample_fields=lambda sample, *args: { - 'entity:sample_id': sample.individual.individual_id, + get_additional_sample_fields=lambda individual, *args: { + 'entity:sample_id': individual.individual_id, 'sequencing_center': 'Broad', }, family_fields={'phenotype_group': { @@ -382,11 +382,11 @@ def gregor_export(request): grouped_data_type_individuals = _get_individual_data_types(projects) # If multiple individual records, prefer WGS - individual_lookup = { + individual_lookup = [ next(data_type_individuals[data_type.upper()] for data_type in GREGOR_DATA_TYPES - if data_type_individuals.get(data_type.upper())): None + if data_type_individuals.get(data_type.upper())) for data_type_individuals in grouped_data_type_individuals.values() - } + ] participant_rows = [] family_map = {} @@ -405,7 +405,7 @@ def _add_row(row, family_id, row_type): parse_anvil_metadata( projects, user=request.user, - individual_samples=individual_lookup, + individuals=individual_lookup, individual_data_types=grouped_data_type_individuals, add_row=_add_row, format_id=_format_gregor_id, @@ -992,7 +992,8 @@ def variant_metadata(request, project_guid): individuals = Individual.objects.filter( family__project__in=projects, family__savedvariant__varianttag__variant_tag_type__category=DISCOVERY_CATEGORY, ).distinct().annotate( - data_types=ArrayAgg('sample__sample_type', distinct=True, filter=Q(sample__isnull=False)) + active_data_types=ArrayAgg('active_datasets__sample_type', distinct=True), + inactive_data_types=ArrayAgg('inactive_datasets__sample_type', distinct=True), ) families_by_id = {} @@ -1018,8 +1019,8 @@ def _add_row(row, family_id, row_type): parse_anvil_metadata( projects, user=request.user, - individual_samples={i: None for i in individuals}, - individual_data_types={i.individual_id: i.data_types for i in individuals}, + individuals=individuals, + individual_data_types={i.individual_id: {*i.active_data_types, *i.inactive_data_types} for i in individuals}, add_row=_add_row, mme_value=ArrayAgg('matchmakersubmissiongenes__saved_variant__variant_id'), include_family_name_display=True, diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index eefe6863bb..92d4a6da0c 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -1,8 +1,8 @@ from collections import defaultdict from datetime import datetime -from django.db.models import F, Q, Value, Case, When, CharField, Aggregate -from django.db.models.functions import Replace +from django.db.models import F, Q, Value, Case, When, CharField, Aggregate, Min +from django.db.models.functions import Replace, Coalesce from django.contrib.auth.models import User from django.contrib.postgres.aggregates import ArrayAgg import requests @@ -163,31 +163,31 @@ def _get_variant_json(saved_variants, fields, annotations): def parse_anvil_metadata( projects: Iterable[Project], user: User, add_row: Callable[[dict, str, str], None], max_loaded_date: str = None, family_fields: dict = None, format_id: Callable[[str], str] = lambda s: s, - get_additional_sample_fields: Callable[[Sample, dict], dict] = None, + get_additional_sample_fields: Callable[[Individual, dict], dict] = None, get_additional_individual_fields: Callable[[Individual, dict], dict] = None, - individual_samples: dict[Individual, Sample] = None, individual_data_types: dict[str, Iterable[str]] = None, + individuals: dict[Individual, Sample] = None, individual_data_types: dict[str, Iterable[str]] = None, airtable_fields: Iterable[str] = None, mme_value: Aggregate = None, get_variant_json: Callable[[Iterable[SavedVariant], list[str], dict], dict] = _get_variant_json, post_process_variant: Callable[[dict, list[dict]], dict] = None, include_no_individual_families: bool = False, omit_airtable: bool = False, include_family_name_display: bool = False, include_family_sample_metadata: bool = False, include_discovery_sample_id: bool = False, include_mondo: bool = False, omit_parent_mnvs: bool = False, proband_only_variants: bool = False): - individual_samples = individual_samples or (_get_loaded_before_date_project_individual_samples(projects, max_loaded_date) \ - if max_loaded_date else _get_all_project_individual_samples(projects)) + if not individuals: + individuals = _get_sample_annotated_individuals(projects, max_loaded_date) family_data_by_id = _get_family_metadata( - {'project__in': projects} if include_no_individual_families else {'individual__in': individual_samples}, + {'project__in': projects} if include_no_individual_families else {'individual__in': individuals}, family_fields, include_family_name_display, include_family_sample_metadata, include_mondo, format_id ) individuals_by_family_id = defaultdict(list) individual_ids_map = {} sample_ids = set() - for individual, sample in individual_samples.items(): + for individual in individuals: individuals_by_family_id[individual.family_id].append(individual) individual_ids_map[individual.id] = (individual.individual_id, individual.guid) - if sample: - sample_ids.add(sample.sample_id) + if getattr(individuals, 'sample_type', None): + sample_ids.add(individual.individual_id) saved_variants_by_family = _get_parsed_saved_discovery_variants_by_family( list(family_data_by_id.keys()), bool(mme_value), get_variant_json, @@ -199,7 +199,7 @@ def parse_anvil_metadata( list(sample_ids) or [i[0] for i in individual_ids_map.values()], user, airtable_fields) matchmaker_individuals = {m['individual_id']: m['value'] for m in MatchmakerSubmission.objects.filter( - individual__in=individual_samples).values('individual_id', value=mme_value)} if mme_value else {} + individual__in=individuals).values('individual_id', value=mme_value)} if mme_value else {} for family_id, family_subject_row in family_data_by_id.items(): saved_variants = saved_variants_by_family[family_id] @@ -223,15 +223,14 @@ def parse_anvil_metadata( add_row(family_row, family_id, FAMILY_ROW_TYPE) for individual in family_individuals: - sample = individual_samples[individual] - airtable_metadata = None has_dbgap_submission = None + sample_type = getattr(individual, 'sample_type', None) if sample_airtable_metadata is not None: - if sample: - airtable_metadata = sample_airtable_metadata.get(sample.sample_id, {}) + if sample_type: + airtable_metadata = sample_airtable_metadata.get(individual.individual_id, {}) dbgap_submission = airtable_metadata.get('dbgap_submission') or set() - has_dbgap_submission = sample.sample_type in dbgap_submission + has_dbgap_submission = sample_type in dbgap_submission elif not sample_ids: airtable_metadata = sample_airtable_metadata.get(individual.individual_id, {}) @@ -249,8 +248,8 @@ def parse_anvil_metadata( add_row(subject_row, family_id, SUBJECT_ROW_TYPE) participant_id = subject_row['participant_id'] - if sample: - sample_row = _get_sample_row(sample, participant_id, has_dbgap_submission, airtable_metadata, include_family_sample_metadata, get_additional_sample_fields) + if sample_type: + sample_row = _get_sample_row(individual, participant_id, has_dbgap_submission, airtable_metadata, include_family_sample_metadata, get_additional_sample_fields) add_row(sample_row, family_id, SAMPLE_ROW_TYPE) if proband_only_variants and individual.proband_relationship != Individual.SELF_RELATIONSHIP: @@ -260,7 +259,7 @@ def parse_anvil_metadata( format_id=format_id, omit_parent_mnvs=omit_parent_mnvs, individual_data_types=(individual_data_types or {}).get(participant_id), family_individuals=family_individuals if proband_only_variants else None, - sample=sample if include_discovery_sample_id else None, + sample_id=individual.individual_id if include_discovery_sample_id else None, post_process_variant=post_process_variant, ) add_row(discovery_row, family_id, DISCOVERY_ROW_TYPE) @@ -270,18 +269,24 @@ def _get_nested_variant_name(v): return v['sv_name'] or f"{v['chrom']}-{v['pos']}-{v['ref']}-{v['alt']}" -def _get_loaded_before_date_project_individual_samples(projects, max_loaded_date): - max_loaded_date = datetime.strptime(max_loaded_date, '%Y-%m-%d') - loaded_samples = _get_sorted_search_samples(projects).filter( - loaded_date__lte=max_loaded_date).select_related('individual') - # Only return the oldest sample for each individual - return {sample.individual: sample for sample in loaded_samples} - - -def _get_all_project_individual_samples(projects): - samples_by_individual_id = {s.individual_id: s for s in _get_sorted_search_samples(projects)} +def _get_sample_annotated_individuals(projects, max_loaded_date): individuals = Individual.objects.filter(family__project__in=projects) - return {i: samples_by_individual_id.get(i.id) for i in individuals} + active_filter = inactive_filter = None + if max_loaded_date: + max_loaded_date = datetime.strptime(max_loaded_date, '%Y-%m-%d') + active_filter = Q(active_datasets__loaded_date__lte=max_loaded_date) + inactive_filter = Q(inactive_datasets__loaded_date__lte=max_loaded_date) + individuals = individuals.filter(active_filter | inactive_filter) + return individuals.annotate( + sample_type=Coalesce( + Min('active_datasets__sample_type', filter=active_filter), + Min('inactive_datasets__sample_type', filter=inactive_filter), + ), + loaded_date=Coalesce( + Min('inactive_datasets__loaded_date', filter=inactive_filter), + Min('active_datasets__loaded_date', filter=active_filter), + ), + ) def _get_sorted_search_samples(projects): @@ -458,27 +463,27 @@ def anvil_export_airtable_fields(airtable_metadata, has_dbgap_submission): } -def _get_sample_row(sample, participant_id, has_dbgap_submission, airtable_metadata, include_family_sample_metadata, get_additional_sample_fields=None): +def _get_sample_row(individual, participant_id, has_dbgap_submission, airtable_metadata, include_family_sample_metadata, get_additional_sample_fields=None): sample_row = { 'participant_id': participant_id, - 'sample_id': sample.sample_id, + 'sample_id': individual.individual_id, } if has_dbgap_submission: sample_row['dbgap_sample_id'] = airtable_metadata.get('dbgap_sample_id', '') if include_family_sample_metadata: sample_row.update({ - 'data_type': sample.sample_type, - 'date_data_generation': sample.loaded_date.strftime('%Y-%m-%d'), + 'data_type': individual.sample_type, + 'date_data_generation': individual.loaded_date.strftime('%Y-%m-%d'), }) if get_additional_sample_fields: - sample_row.update(get_additional_sample_fields(sample, airtable_metadata)) + sample_row.update(get_additional_sample_fields(individual, airtable_metadata)) return sample_row def _get_genetic_findings_rows(rows: list[dict], individual: Individual, family_row: dict, participant_id: str, individual_data_types: Iterable[str], family_individuals: dict[str, str], post_process_variant: Callable[[dict, list[dict]], dict], - format_id: Callable[[str], str], omit_parent_mnvs: bool, sample: Sample) -> list[dict]: + format_id: Callable[[str], str], omit_parent_mnvs: bool, sample_id: str) -> list[dict]: parsed_rows = [] variants_by_gene = defaultdict(list) for row in (rows or []): @@ -512,8 +517,8 @@ def _get_genetic_findings_rows(rows: list[dict], individual: Individual, family_ parsed_row['method_of_discovery'] = '|'.join([ METHOD_MAP.get(data_type) for data_type in individual_data_types if data_type in Dataset.SAMPLE_TYPE_LOOKUP ]) - if sample is not None: - parsed_row['sample_id'] = sample.sample_id + if sample_id is not None: + parsed_row['sample_id'] = sample_id parsed_rows.append(parsed_row) variants_by_gene[row[GENE_COLUMN]].append({**parsed_row, 'individual_genotype': individual_genotype}) From a2714e34625a78f103047817202efcbac933044b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 6 Apr 2026 16:25:14 -0400 Subject: [PATCH 007/119] fix report tetss --- seqr/views/apis/report_api.py | 8 ++++---- seqr/views/apis/report_api_tests.py | 6 +++--- seqr/views/utils/anvil_metadata_utils.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index 1bbef12668..479041d5e3 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -518,14 +518,14 @@ def _process_participant_row(participant, phenotype_rows, missing_participant_id def _get_individual_data_types(projects): rna_individuals = set(RnaSample.objects.filter(individual__family__project__in=projects).values_list('individual_id', flat=True)) individuals = Individual.objects.filter(family__project__in=projects).annotate( - sample_types=ArrayAgg('active_datasets__sample_type', distinct=True), - ).prefetch_related( - 'family__project', 'mother', 'father') + active_sample_types=ArrayAgg('active_datasets__sample_type', distinct=True, filter=Q(active_datasets__isnull=False)), + inactive_sample_types=ArrayAgg('inactive_datasets__sample_type', distinct=True, filter=Q(inactive_datasets__isnull=False)), + ).prefetch_related('family__project', 'mother', 'father') grouped_data_type_individuals = defaultdict(dict) for i in individuals: participant_id = _format_gregor_id(i.individual_id) - data_types = set(i.sample_types) | ({'RNA'} if i.id in rna_individuals else set()) + data_types = {*i.active_sample_types, *i.inactive_sample_types} | ({'RNA'} if i.id in rna_individuals else set()) if data_types: grouped_data_type_individuals[participant_id].update({data_type: i for data_type in data_types}) return grouped_data_type_individuals diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index e52abc78c1..bcb3075648 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -18,7 +18,7 @@ "id": "rec2B6OGmQpAkQW3s", "fields": { "SeqrCollaboratorSampleID": "VCGS_FAM203_621_D1", - "CollaboratorSampleID": "NA19675", + "CollaboratorSampleID": "NA19675_1", "Collaborator": ["recW24C2CJW5lT64K"], "dbgap_study_id": "dbgap_stady_id_1", "dbgap_subject_id": "dbgap_subject_id_1", @@ -746,7 +746,7 @@ def _check_anvil_export_response(self, response, mock_zip, no_analyst_project_ur 'entity:sample_id', '01-subject_id', '02-sample_id', '03-dbgap_sample_id', '04-sequencing_center', '05-sample_source', '06-tissue_affected_status',]) self.assertIn( - ['NA19675_1', 'NA19675_1', 'NA19675', 'SM-A4GQ4', 'Broad', '-', '-'], + ['NA19675_1', 'NA19675_1', 'NA19675_1', 'SM-A4GQ4', 'Broad', '-', '-'], sample_file, ) @@ -767,7 +767,7 @@ def _check_anvil_export_response(self, response, mock_zip, no_analyst_project_ur '1_248367227_HG00731', 'HG00731', 'HG00731', 'RP11', 'Known', 'paternal', 'Homozygous', 'GRCh37', '1', '248367227', 'TC', 'T', '-', '-', 'ENST00000371839', '-', '-', '-', '-'], discovery_file) self.assertIn([ - '21_3343353_NA19675_1', 'NA19675_1', 'NA19675', 'RP11', 'Candidate', 'de novo', + '21_3343353_NA19675_1', 'NA19675_1', 'NA19675_1', 'RP11', 'Candidate', 'de novo', 'Heterozygous', 'GRCh37', '21', '3343353', 'GAGA', 'G', 'c.375_377delTCT', 'p.Leu126del', 'ENST00000258436.5', '-', '-', '-', 'This individual is published in PMID34415322'], discovery_file) self.assertIn([ diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 92d4a6da0c..e8a1d9598a 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -186,7 +186,7 @@ def parse_anvil_metadata( for individual in individuals: individuals_by_family_id[individual.family_id].append(individual) individual_ids_map[individual.id] = (individual.individual_id, individual.guid) - if getattr(individuals, 'sample_type', None): + if getattr(individual, 'sample_type', None): sample_ids.add(individual.individual_id) saved_variants_by_family = _get_parsed_saved_discovery_variants_by_family( From 84e6b74acfdfa9befb3d011b18d7a35e3cf26136 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 7 Apr 2026 10:57:05 -0400 Subject: [PATCH 008/119] update dashboard api --- seqr/fixtures/1kg_project.json | 17 ++++++++++++++++- seqr/views/apis/dashboard_api.py | 24 +++++++++++++----------- seqr/views/apis/report_api.py | 2 +- seqr/views/utils/anvil_metadata_utils.py | 2 +- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index f1216876a1..839da1a3bc 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -963,7 +963,7 @@ "created_by": null, "last_modified_date": "2017-03-13T09:07:50.052Z", "sample_type": "WES", - "inactive_individuals": [8, 10, 11, 12, 14, 17], + "inactive_individuals": [8, 10, 11, 12, 14], "dataset_type": "SNV_INDEL", "data_source": "1kg.vcf.gz", "loaded_date": "2017-02-05T06:20:55.397Z" @@ -1074,6 +1074,21 @@ "data_type": "S" } }, +{ + "model": "seqr.dataset", + "pk": 154, + "fields": { + "guid": "S000154_na20889", + "created_date": "2017-02-05T06:42:55.397Z", + "created_by": null, + "last_modified_date": "2017-03-13T09:07:50.277Z", + "sample_type": "WES", + "inactive_individuals": [17], + "dataset_type": "SNV_INDEL", + "data_source": "auto__2023-08-08", + "loaded_date": "2017-02-05T06:28:55.397Z" + } +}, { "model": "seqr.rnasample", "pk": 161, diff --git a/seqr/views/apis/dashboard_api.py b/seqr/views/apis/dashboard_api.py index efd476700c..f4a0ceb6bd 100644 --- a/seqr/views/apis/dashboard_api.py +++ b/seqr/views/apis/dashboard_api.py @@ -2,8 +2,9 @@ APIs used by the main seqr dashboard page """ from django.db import models +from django.db.models.functions import Coalesce -from seqr.models import ProjectCategory, Sample, Dataset, RnaSample, Family, Project +from seqr.models import ProjectCategory, Individual, Dataset, RnaSample, Family, Project from seqr.views.utils.individual_utils import check_project_individuals_deletable from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.orm_to_json_utils import get_json_for_projects @@ -59,13 +60,19 @@ def _get_projects_json(user): projects_by_guid[project_guid]['analysisStatusCounts'] = {} projects_by_guid[project_guid]['analysisStatusCounts'][agg['analysis_status']] = agg['count'] - sample_type_status_counts = _sample_type_counts( - Sample.objects.filter(individual__family__project__in=projects, dataset_type=Dataset.DATASET_TYPE_VARIANT_CALLS) - ) + _sample_type_counts( - RnaSample.objects.filter(individual__family__project__in=projects).annotate(sample_type=models.Value('RNA')) + sample_type_status_counts = list( + Individual.objects.filter(family__project__in=projects).annotate(sample_type=Coalesce( + models.F('active_datasets__sample_type'), models.F('inactive_datasets__sample_type'), + )).filter(sample_type__isnull=False).values('sample_type', project_guid=models.F('family__project__guid')).annotate( + count=models.Count('id', distinct=True) + ) + ) + list( + RnaSample.objects.filter(individual__family__project__in=projects).values( + project_guid=models.F('individual__family__project__guid') + ).annotate(sample_type=models.Value('RNA'), count=models.Count('individual_id', distinct=True)) ) for agg in sample_type_status_counts: - project_guid = agg['individual__family__project__guid'] + project_guid = agg['project_guid'] if 'sampleTypeCounts' not in projects_by_guid[project_guid]: projects_by_guid[project_guid]['sampleTypeCounts'] = {} projects_by_guid[project_guid]['sampleTypeCounts'][agg['sample_type']] = agg['count'] @@ -73,11 +80,6 @@ def _get_projects_json(user): return projects_by_guid -def _sample_type_counts(sample_q): - return list(sample_q.values( - 'individual__family__project__guid', 'sample_type', - ).annotate(count=models.Count('individual_id', distinct=True))) - def _retrieve_project_categories_by_guid(project_guids): """Retrieves project categories from the database, and returns a 'project_categories_by_guid' dictionary, while also adding a 'projectCategoryGuids' attribute to each project dict in 'projects_by_guid'. diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index 479041d5e3..efd690e80d 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -1,7 +1,7 @@ from collections import defaultdict from datetime import datetime, timedelta -from django.db.models import Count, Q, F, Sum, Value +from django.db.models import Count, Q, F, Value from django.contrib.postgres.aggregates import ArrayAgg import json import re diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index e8a1d9598a..3e30ea13df 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -165,7 +165,7 @@ def parse_anvil_metadata( max_loaded_date: str = None, family_fields: dict = None, format_id: Callable[[str], str] = lambda s: s, get_additional_sample_fields: Callable[[Individual, dict], dict] = None, get_additional_individual_fields: Callable[[Individual, dict], dict] = None, - individuals: dict[Individual, Sample] = None, individual_data_types: dict[str, Iterable[str]] = None, + individuals: list[Individual] = None, individual_data_types: dict[str, Iterable[str]] = None, airtable_fields: Iterable[str] = None, mme_value: Aggregate = None, get_variant_json: Callable[[Iterable[SavedVariant], list[str], dict], dict] = _get_variant_json, post_process_variant: Callable[[dict, list[dict]], dict] = None, include_no_individual_families: bool = False, omit_airtable: bool = False, include_family_name_display: bool = False, include_family_sample_metadata: bool = False, From 620e8e98c7d2d5c3445fc565f2daee1782801d8f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 7 Apr 2026 13:19:35 -0400 Subject: [PATCH 009/119] update pedigree validation --- seqr/views/utils/pedigree_info_utils.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/seqr/views/utils/pedigree_info_utils.py b/seqr/views/utils/pedigree_info_utils.py index f29f8a2cc1..b87573e76a 100644 --- a/seqr/views/utils/pedigree_info_utils.py +++ b/seqr/views/utils/pedigree_info_utils.py @@ -15,7 +15,7 @@ from seqr.utils.middleware import ErrorsWarningsException from seqr.views.utils.json_utils import _to_snake_case, _to_title_case from seqr.views.utils.permissions_utils import user_is_pm, get_pm_user_emails -from seqr.models import Individual, Sample +from seqr.models import Individual logger = SeqrLogger(__name__) @@ -361,22 +361,24 @@ def get_validated_related_individuals(project, records_by_id, errors, related_gu affected = records_by_id[individual_id].get(JsonConstants.AFFECTED_COLUMN, Individual.AFFECTED_STATUS_UNKNOWN) affected_status_by_family[family_id].append(affected) - search_samples = Sample.objects.filter(individual__family__project=project, is_active=True) + search_individuals = Individual.objects.filter(family__project=project) sample_type = None if search_dataset_type: - search_samples = search_samples.filter(dataset_type=search_dataset_type) + search_individuals = search_individuals.filter(active_datasets__dataset_type=search_dataset_type) if search_sample_type: - search_samples = search_samples.filter(sample_type=search_sample_type) - elif search_samples: - sample_type = search_samples.first().sample_type + search_individuals = search_individuals.filter(active_datasets__sample_type=search_sample_type) + elif search_individuals: + sample_type = search_individuals.first().active_datasets.first().sample_type + else: + search_individuals = search_individuals.filter(active_datasets__dataset_type__isnull=False) previous_loaded_individuals = { i[JsonConstants.INDIVIDUAL_ID_COLUMN]: i - for i in search_samples.values( - 'individual_id', **{ - JsonConstants.INDIVIDUAL_ID_COLUMN: F('individual__individual_id'), - JsonConstants.FAMILY_ID_COLUMN: F('individual__family__family_id'), - }) + for i in search_individuals.values( + **{ + JsonConstants.INDIVIDUAL_ID_COLUMN: F('individual_id'), + JsonConstants.FAMILY_ID_COLUMN: F('family__family_id'), + }).annotate(individual_id=F('id')) } if validate_expected_samples: From 5dd40f98fd7b323917a837b3fda98c9883edebe1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 7 Apr 2026 13:57:20 -0400 Subject: [PATCH 010/119] fix delete individual --- seqr/views/apis/family_api_tests.py | 6 ++++-- seqr/views/apis/individual_api_tests.py | 12 ++++++++++-- seqr/views/apis/project_api.py | 4 +--- seqr/views/utils/individual_utils.py | 14 +++++++++++--- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index 67fc881917..847d190666 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- import json +import pdb + import mock from copy import deepcopy from datetime import datetime @@ -16,7 +18,7 @@ FAMILY_NOTE_FIELDS, FAMILY_FIELDS, IGV_SAMPLE_FIELDS, \ SAMPLE_FIELDS, INDIVIDUAL_FIELDS, INTERNAL_INDIVIDUAL_FIELDS, INTERNAL_FAMILY_FIELDS, CASE_REVIEW_FAMILY_FIELDS, \ MATCHMAKER_SUBMISSION_FIELDS, TAG_TYPE_FIELDS, CASE_REVIEW_INDIVIDUAL_FIELDS -from seqr.models import FamilyAnalysedBy, AnalysisGroup, Sample +from seqr.models import FamilyAnalysedBy, AnalysisGroup, Individual FAMILY_GUID = 'F000001_1' FAMILY_GUID2 = 'F000002_2' @@ -324,7 +326,7 @@ def test_delete_families_handler(self, mock_pm_group, mock_internal_namespaces): # Test success MatchmakerSubmission.objects.update(deleted_date=datetime.now()) - Sample.objects.update(is_active=False) + Individual.active_datasets.through.objects.all().delete() response = self.client.post(url, content_type='application/json', data=json.dumps(req_values)) self.assertEqual(response.status_code, 200) diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 94cb1c80fa..3985bf458d 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -12,7 +12,7 @@ from io import BytesIO from openpyxl import load_workbook -from seqr.models import Individual, Sample, SavedVariant, VariantTag +from seqr.models import Individual, Sample, Dataset, SavedVariant, VariantTag from seqr.views.apis.individual_api import edit_individuals_handler, update_individual_handler, \ delete_individuals_handler, receive_individuals_table_handler, save_individuals_table_handler, \ receive_individuals_metadata_handler, save_individuals_metadata_table_handler, update_individual_hpo_terms, \ @@ -369,7 +369,9 @@ def test_delete_individuals(self, mock_pm_group): })) self.assertEqual(response.status_code, 400) self.assertListEqual(response.json()['errors'], ['Unable to delete individuals with active search sample: NA19678']) - Sample.objects.filter(guid__in=['S000130_na19678', 'S000143_na20885', 'S000173_na21987']).update(is_active=False) + for dataset in Dataset.objects.filter(guid__in=['S000130_na19678', 'S000143_na20885', 'S000147_na21234']): + dataset.inactive_individuals.set(dataset.active_individuals.all()) + dataset.active_individuals.set(set()) # send valid requests response = self.client.post(individuals_url, content_type='application/json', data=json.dumps({ @@ -385,6 +387,7 @@ def test_delete_individuals(self, mock_pm_group): }) self.assertFalse('I000002_na19678' in response_json['familiesByGuid']['F000001_1']['individualGuids']) self.assertIsNone(response_json['familiesByGuid']['F000001_1']['pedigreeImage']) + self.assertFalse(Dataset.objects.filter(guid='S000130_na19678').exists()) # Test PM permission pm_required_delete_individuals_url = reverse(delete_individuals_handler, args=[PM_REQUIRED_PROJECT_GUID]) @@ -433,6 +436,11 @@ def test_delete_individuals(self, mock_pm_group): 'individuals': [EXTERNAL_WORKSPACE_INDIVIDUAL_UPDATE_DATA] })) self.assertEqual(response.status_code, 200 if self.HAS_EXTERNAL_PROJECT_ACCESS else 403) + if self.HAS_EXTERNAL_PROJECT_ACCESS: + self.assertListEqual( + list(Dataset.objects.get(guid='S000147_na21234').inactive_individuals.order_by('id').values_list('guid', flat=True)), + ['I000018_na21234', 'I000021_na21654'], + ) def test_individuals_table_handler_errors(self): individuals_url = reverse(receive_individuals_table_handler, args=[PROJECT_GUID]) diff --git a/seqr/views/apis/project_api.py b/seqr/views/apis/project_api.py index b5793f8db7..31cb7dcd92 100644 --- a/seqr/views/apis/project_api.py +++ b/seqr/views/apis/project_api.py @@ -448,9 +448,7 @@ def _delete_project(project_guid, user): project = Project.objects.get(guid=project_guid) check_user_created_object_permissions(project, user) - individual_guids_to_delete = Individual.objects.filter( - family__project__guid=project_guid).values_list('guid', flat=True) - delete_individuals(project, individual_guids_to_delete, user) + delete_individuals(project, individual_guids=None, user=user) Family.bulk_delete(user, project=project) diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index b9e46d1a21..816b58ed4a 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -5,7 +5,7 @@ from clickhouse_search.models.postgres_dicts import SexDict from matchmaker.models import MatchmakerSubmission, MatchmakerResult -from seqr.models import Sample, IgvSample, RnaSample, Individual, Family, FamilyNote +from seqr.models import Dataset, IgvSample, RnaSample, Individual, Family, FamilyNote from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.search.add_data_utils import trigger_rebuild_gt_stats from seqr.views.utils.json_to_orm_utils import update_individual_from_json, update_individual_parents, create_model_from_json, \ @@ -193,7 +193,15 @@ def delete_individuals(project, individual_guids, user): if errors: raise ErrorsWarningsException(errors) - Sample.bulk_delete(user, individual__in=individuals_to_delete) + datasets = Dataset.objects.filter(inactive_individuals__in=individuals_to_delete) + if individual_guids is None: + Dataset.bulk_delete(user, queryset=datasets) + else: + for dataset in datasets: + dataset.inactive_individuals.remove(*individuals_to_delete) + if not dataset.inactive_individuals.exists(): + dataset.delete_model(user, user_can_delete=True) + IgvSample.bulk_delete(user, individual__in=individuals_to_delete) RnaSample.bulk_delete(user, individual__in=individuals_to_delete) MatchmakerResult.bulk_delete(user, submission__individual__in=individuals_to_delete, submission__deleted_date__isnull=False) @@ -231,7 +239,7 @@ def _validate_no_sumissions_no_search_samples(individuals_to_delete): individuals_to_delete, 'MME submission', dict(matchmakersubmission__isnull=False, matchmakersubmission__deleted_date__isnull=True) ) + _validate_delete_individuals( - individuals_to_delete, 'search sample', dict(sample__is_active=True) + individuals_to_delete, 'search sample', dict(active_datasets__isnull=False) ) From 19d480ae6d98db7ed97aaaf1abecaef401918b25 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 7 Apr 2026 14:16:12 -0400 Subject: [PATCH 011/119] clean up --- clickhouse_search/search_tests.py | 12 ++++++------ seqr/views/apis/data_manager_api.py | 15 ++++++++++----- seqr/views/apis/data_manager_api_tests.py | 5 ++--- seqr/views/apis/individual_api_tests.py | 6 ++++-- seqr/views/utils/anvil_metadata_utils.py | 6 +----- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index a5078a8996..c49d0faff7 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -38,7 +38,7 @@ DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, \ format_cached_variant from reference_data.models import Omim -from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults, SavedVariant, Individual +from seqr.models import Project, Family, Sample, Dataset, VariantSearch, VariantSearchResults, SavedVariant, Individual from seqr.views.apis.data_manager_api import trigger_delete_project from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, \ SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \ @@ -2442,9 +2442,9 @@ def test_trigger_delete_project(self): 23: {'ac_wes': 0, 'ac_wgs': 0, 'hom_wes': 0, 'hom_wgs': 0, 'ac_affected': 0, 'hom_affected': 0}, }) - project_samples = Sample.objects.filter(individual__family__project__guid='R0001_1kg', is_active=True) - self.assertEqual(project_samples.filter(dataset_type='SNV_INDEL').count(), 0) - self.assertEqual(project_samples.count(), 4) + project_datasets = Dataset.objects.filter(active_individuals__family__project__guid='R0001_1kg') + self.assertEqual(project_datasets.filter(dataset_type='SNV_INDEL').count(), 0) + self.assertEqual(project_datasets.count(), 4) body['datasetType'] = 'SV' response = self.client.post(url, content_type='application/json', data=json.dumps(body)) @@ -2455,7 +2455,7 @@ def test_trigger_delete_project(self): 'Deleted all GCNV search data for project 1kg project n\xe5me with uni\xe7\xf8de', ], }) - self.assertEqual(project_samples.filter(dataset_type='SV').count(), 0) - self.assertEqual(project_samples.count(), 1) + self.assertEqual(project_datasets.filter(dataset_type='SV').count(), 0) + self.assertEqual(project_datasets.count(), 1) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index fb3f8799b1..e40f47ade4 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -23,7 +23,7 @@ from seqr.views.utils.permissions_utils import data_manager_required, pm_or_data_manager_required, get_internal_projects from seqr.views.utils.terra_api_utils import anvil_enabled -from seqr.models import Sample, Dataset, RnaSample, Individual, Project, PhenotypePrioritization +from seqr.models import Dataset, RnaSample, Individual, Project, PhenotypePrioritization from settings import LOADING_DATASETS_DIR, LUIGI_UI_SERVICE_HOSTNAME, LUIGI_UI_SERVICE_PORT @@ -400,12 +400,17 @@ def trigger_delete_project(request): project_guid = request_json.pop('project') dataset_type = request_json.get('datasetType') project = Project.objects.get(guid=project_guid) - samples = Sample.objects.filter(individual__family__project=project, dataset_type=dataset_type, is_active=True) + datasets = Dataset.objects.filter(active_individuals__family__project=project, dataset_type=dataset_type) sample_types = list( - samples.values_list('sample_type', flat=True).distinct() + datasets.values_list('sample_type', flat=True).distinct() ) if dataset_type == Dataset.DATASET_TYPE_SV_CALLS else [None] - updated = Sample.bulk_update(user=request.user, update_json={'is_active': False}, queryset=samples) - info = [f'Deactivated search for {len(updated)} individuals'] + updated = 0 + for dataset in datasets: + active_individuals = dataset.active_individuals.all() + updated += len(active_individuals) + dataset.inactive_individuals.set(active_individuals) + dataset.active_individuals.clear() + info = [f'Deactivated search for {updated} individuals'] for sample_type in sample_types: info.append(delete_clickhouse_project(project, dataset_type=dataset_type, sample_type=sample_type)) return create_json_response({'info': info}) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 3b688ab47e..f1024aa908 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -11,7 +11,7 @@ get_loaded_projects, load_data, trigger_delete_family from seqr.views.utils.orm_to_json_utils import _get_json_for_models from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest -from seqr.models import Individual, Sample, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier, RnaSample, Project, PhenotypePrioritization +from seqr.models import Dataset, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier, RnaSample, Project, PhenotypePrioritization from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL PROJECT_GUID = 'R0001_1kg' @@ -1279,8 +1279,7 @@ def test_trigger_delete_family(self): ], }) - family_samples = Sample.objects.filter(individual__family_id=2, is_active=True) - self.assertEqual(family_samples.count(), 0) + self.assertEqual(Dataset.objects.filter(active_individuals__family_id=2).count(), 0) self.assertEqual(len(responses.calls), 1) self.assertDictEqual(json.loads(responses.calls[-1].request.body), { diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 3985bf458d..00a0bf7a0f 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -12,7 +12,7 @@ from io import BytesIO from openpyxl import load_workbook -from seqr.models import Individual, Sample, Dataset, SavedVariant, VariantTag +from seqr.models import Individual, Dataset, SavedVariant, VariantTag from seqr.views.apis.individual_api import edit_individuals_handler, update_individual_handler, \ delete_individuals_handler, receive_individuals_table_handler, save_individuals_table_handler, \ receive_individuals_metadata_handler, save_individuals_metadata_table_handler, update_individual_hpo_terms, \ @@ -206,7 +206,9 @@ def test_edit_individuals(self, mock_pm_group): ]) # send valid request - Sample.objects.filter(guid__in=['S000130_na19678', 'S000135_na20870']).update(is_active=False) + for dataset in Dataset.objects.filter(guid__in=['S000130_na19678', 'S000129_na19675']): + dataset.inactive_individuals.set(dataset.active_individuals.all()) + dataset.active_individuals.set(set()) response = self.client.post(edit_individuals_url, content_type='application/json', data=json.dumps({ 'individuals': [INDIVIDUAL_IDS_UPDATE_DATA, INDIVIDUAL_FAMILY_UPDATE_DATA] })) diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 3e30ea13df..53ea3b4bcd 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -10,7 +10,7 @@ from matchmaker.models import MatchmakerSubmission from reference_data.models import HumanPhenotypeOntology, Omim, GENOME_VERSION_LOOKUP -from seqr.models import Project, Family, Individual, Sample, Dataset, SavedVariant, VariantTagType +from seqr.models import Project, Family, Individual, Dataset, SavedVariant, VariantTagType from seqr.views.utils.airtable_utils import AirtableSession from seqr.utils.gene_utils import get_genes from seqr.utils.middleware import ErrorsWarningsException @@ -289,10 +289,6 @@ def _get_sample_annotated_individuals(projects, max_loaded_date): ) -def _get_sorted_search_samples(projects): - return Sample.objects.filter(individual__family__project__in=projects).order_by('-loaded_date') - - HET = 'Heterozygous' HOM_ALT = 'Homozygous' HEMI = 'Hemizygous' From 86ed8d4d59f3bf572ba6d7d8f30f87c3d1de1de8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 7 Apr 2026 15:29:53 -0400 Subject: [PATCH 012/119] fix delete family data --- clickhouse_search/search_tests.py | 1 + .../transfer_families_to_different_project.py | 17 ++++++++++++++- ...fer_families_to_different_project_tests.py | 21 +++++++++++++++---- seqr/utils/search/add_data_utils.py | 13 ++++++++---- seqr/views/apis/data_manager_api.py | 2 +- seqr/views/apis/data_manager_api_tests.py | 2 +- 6 files changed, 45 insertions(+), 11 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index c49d0faff7..8d40ca0198 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -2445,6 +2445,7 @@ def test_trigger_delete_project(self): project_datasets = Dataset.objects.filter(active_individuals__family__project__guid='R0001_1kg') self.assertEqual(project_datasets.filter(dataset_type='SNV_INDEL').count(), 0) self.assertEqual(project_datasets.count(), 4) + self.assertEqual(Dataset.objects.get(guid='S000129_na19675').inactive_individuals.count(), 7) body['datasetType'] = 'SV' response = self.client.post(url, content_type='application/json', data=json.dumps(body)) diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index 356a0a615c..8dea5c4a11 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -1,6 +1,7 @@ from django.core.management.base import BaseCommand +from django.db.models import Q -from seqr.models import Project, Family, VariantTag, VariantTagType +from seqr.models import Project, Family, VariantTag, VariantTagType, Dataset from seqr.utils.search.add_data_utils import trigger_delete_families_search import logging @@ -34,8 +35,22 @@ def handle(self, *args, **options): ] logger.info(f'Skipping {num_found - len(families)} families with analysis groups in the project: {", ".join(group_families)}') + family_guids = list(families.values_list('guid', flat=True)) trigger_delete_families_search(from_project, list(families.values_list('guid', flat=True))) + remaining_families = Family.objects.filter(project=from_project).exclude(id__in={f.id for f in families}) + split_dataset = Dataset.objects.filter(inactive_individuals__family__in=families).filter( + Q(inactive_individuals__family__in=remaining_families) | Q(active_individuals__family__in=remaining_families) + ).distinct() + logger.info(f'Splitting {split_dataset.count()} datasets') + for dataset in split_dataset: + individuals = dataset.inactive_individuals.filter(family__in=families) + dataset.inactive_individuals.remove(*individuals) + new_dataset = dataset + new_dataset.pk = None + new_dataset.save() + new_dataset.inactive_individuals.set(individuals) + for variant_tag_type in VariantTagType.objects.filter(project=from_project): variant_tags = VariantTag.objects.filter(saved_variants__family__in=families, variant_tag_type=variant_tag_type) if variant_tags: diff --git a/seqr/management/tests/transfer_families_to_different_project_tests.py b/seqr/management/tests/transfer_families_to_different_project_tests.py index b2feeecd53..d346420ad0 100644 --- a/seqr/management/tests/transfer_families_to_different_project_tests.py +++ b/seqr/management/tests/transfer_families_to_different_project_tests.py @@ -1,7 +1,7 @@ from django.core.management import call_command import responses -from seqr.models import Family, VariantTagType, VariantTag, Sample +from seqr.models import Family, VariantTagType, VariantTag, Dataset from seqr.views.utils.test_utils import AnvilAuthenticationTestCase @@ -26,6 +26,7 @@ def test_command(self): ('Found 3 out of 4 families. No match for: 12.', None), ('Skipping 1 families with analysis groups in the project: 5 (Test Group 1)', None), *self.LOGS, + ('Splitting 2 datasets', None), ('Updating "Excluded" tags', None), ('Updating families', None), ('Done.', None), @@ -45,9 +46,21 @@ def test_command(self): self.assertEqual(len(new_tags), 1) self.assertEqual(new_tags[0].saved_variants.first().family, family) - samples = Sample.objects.filter(individual__family=family) - self.assertEqual(samples.count(), 7) - self.assertEqual(samples.filter(is_active=True).count(), 0) + existing_dataset = Dataset.objects.get(guid='S000129_na19675') + self.assertListEqual(list(existing_dataset.active_individuals.order_by('id').values_list('id', flat=True)),[1, 7, 9]) + self.assertListEqual(list(existing_dataset.inactive_individuals.order_by('id').values_list('id', flat=True)), [3]) + self.assertEqual(Dataset.objects.filter(active_individuals__family=family).count(), 0) + datasets = {d.guid: d for d in Dataset.objects.filter(inactive_individuals__family=family).distinct()} + self.assertEqual(len(datasets), 3) + previous_guids = {'S000145_hg00731', 'S000149_hg00733'} + self.assertTrue(previous_guids.issubset(set(datasets.keys()))) + split_dataset = next(d for guid, d in datasets.items() if guid not in previous_guids) + self.assertListEqual(list(split_dataset.inactive_individuals.order_by('id').values_list('id', flat=True)), [4, 5, 6]) + self.assertEqual(split_dataset.active_individuals.count(), 0) + self.assertEqual(split_dataset.sample_type, 'WES') + self.assertEqual(split_dataset.dataset_type, 'SNV_INDEL') + self.assertEqual(split_dataset.data_source, 'test_index') + self.assertEqual(split_dataset.loaded_date.isoformat(), '2017-02-05T06:12:55.397000+00:00') family = Family.objects.get(family_id='4') self.assertEqual(family.project.guid, 'R0003_test') diff --git a/seqr/utils/search/add_data_utils.py b/seqr/utils/search/add_data_utils.py index a9de1385b9..43881c22b7 100644 --- a/seqr/utils/search/add_data_utils.py +++ b/seqr/utils/search/add_data_utils.py @@ -64,12 +64,17 @@ def update_airtable_loading_tracking_status(project, status, additional_update=N ) def trigger_delete_families_search(project, family_guids, user=None): - search_samples = Sample.objects.filter(is_active=True, individual__family__guid__in=family_guids) + num_updated = 0 + updated_families = set() + for dataset in Dataset.objects.filter(active_individuals__family__guid__in=family_guids).distinct(): + active_individuals = dataset.active_individuals.filter(family__guid__in=family_guids) + num_updated += len(active_individuals) + updated_families.update(active_individuals.values_list('family__family_id', flat=True).distinct()) + dataset.inactive_individuals.add(*active_individuals) + dataset.active_individuals.remove(*active_individuals) info = [] - if search_samples: - updated_families = search_samples.values_list("individual__family__family_id", flat=True).distinct() + if num_updated: family_summary = ", ".join(sorted(updated_families)) - num_updated = search_samples.update(is_active=False) message = f'Disabled search for {num_updated} samples in the following {len(updated_families)} families: {family_summary}' info.append(message) logger.info(message, user) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index e40f47ade4..23e7fa80e2 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -408,7 +408,7 @@ def trigger_delete_project(request): for dataset in datasets: active_individuals = dataset.active_individuals.all() updated += len(active_individuals) - dataset.inactive_individuals.set(active_individuals) + dataset.inactive_individuals.add(*active_individuals) dataset.active_individuals.clear() info = [f'Deactivated search for {updated} individuals'] for sample_type in sample_types: diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index f1024aa908..cd6681fd08 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -11,7 +11,7 @@ get_loaded_projects, load_data, trigger_delete_family from seqr.views.utils.orm_to_json_utils import _get_json_for_models from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest -from seqr.models import Dataset, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier, RnaSample, Project, PhenotypePrioritization +from seqr.models import Dataset, Individual, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier, RnaSample, Project, PhenotypePrioritization from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL PROJECT_GUID = 'R0001_1kg' From 0bbfee8e1e1a010e276af7435df36492fd405ad7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 7 Apr 2026 15:54:26 -0400 Subject: [PATCH 013/119] update search families --- clickhouse_search/search.py | 68 +++++++++++++++---------------- clickhouse_search/search_tests.py | 18 ++++---- 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 561d602a5c..1f2824b19d 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -16,7 +16,7 @@ from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, \ ENTRY_CLASS_MAP, VARIANTS_CLASS_MAP, VARIANT_DETAILS_CLASS_MAP from reference_data.models import GeneInfo, GeneConstraint, Omim, GENOME_VERSION_LOOKUP -from seqr.models import Sample, Dataset, PhenotypePrioritization, Family, Individual +from seqr.models import Dataset, PhenotypePrioritization, Family, Individual from seqr.utils.logging_utils import SeqrLogger from clickhouse_search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \ PRIORITIZED_GENE_SORT, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS, RECESSIVE, AFFECTED, MALE_SEXES, \ @@ -378,9 +378,9 @@ def _add_individual_guids(results, encode_genotypes_json=False): for r in (result if isinstance(result, list) else [result]): families.update(r.get('familyGenotypes', {}).keys()) sample_map = { - (family_guid, sample_id): individual_guid for family_guid, individual_guid, sample_id in Sample.objects.filter( - individual__family__guid__in=families, is_active=True, - ).values_list('individual__family__guid', 'individual__guid', 'sample_id') + (family_guid, sample_id): individual_guid for family_guid, individual_guid, sample_id in Individual.objects.filter( + family__guid__in=families, active_datasets__isnull=False, + ).values_list('family__guid', 'guid', 'individual_id') } for result in results: if isinstance(result, list): @@ -502,66 +502,66 @@ def _is_matched_minimal_transcript(transcript, minimal_transcript): def _get_valid_samples(families, dataset_type, sample_type, allow_no_samples): - samples = Sample.objects.filter(individual__family__in=families, is_active=True) - if not samples.exists(): + individuals = Individual.objects.filter(family__in=families, active_datasets__isnull=False) + if not individuals.exists(): if allow_no_samples: return None raise InvalidSearchException(f'No search data found for families {", ".join([f.family_id for f in families])}') - samples = samples.filter(dataset_type=dataset_type) + individuals = individuals.filter(active_datasets__dataset_type=dataset_type) if sample_type: - samples = samples.filter(sample_type=sample_type) + individuals = individuals.filter(active_datasets__sample_type=sample_type) - mismatch_affected_samples = samples.values('sample_id').annotate( - projects=ArrayAgg('individual__family__project__name', distinct=True), - affected=ArrayAgg('individual__affected', distinct=True), + mismatch_affected_samples = individuals.values('individual_id').annotate( + projects=ArrayAgg('family__project__name', distinct=True), + affected=ArrayAgg('affected', distinct=True), ).filter(affected__len__gt=1) if mismatch_affected_samples: raise InvalidSearchException( 'The following samples are incorrectly configured and have different affected statuses in different projects: ' + - ', '.join([f'{agg["sample_id"]} ({"/ ".join(agg["projects"])})' for agg in mismatch_affected_samples]), + ', '.join([f'{agg["individual_id"]} ({"/ ".join(agg["projects"])})' for agg in mismatch_affected_samples]), ) - return samples + return individuals -def _get_sample_metadata(samples, affected_family_only, annotate_affected_males): - skip_individual_guid = samples.values('individual__family__project_id').distinct().count() > 1 +def _get_sample_metadata(individuals, affected_family_only, annotate_affected_males): + skip_individual_guid = individuals.values('family__project_id').distinct().count() > 1 family_array_kwargs = {'distinct': True} if affected_family_only: - family_array_kwargs['filter'] = Q(individual__affected=Individual.AFFECTED_STATUS_AFFECTED) + family_array_kwargs['filter'] = Q(affected=Individual.AFFECTED_STATUS_AFFECTED) annotations = { - 'project_guids': ArrayAgg('individual__family__project__guid', distinct=True), - 'family_guids': ArrayAgg('individual__family__guid', **family_array_kwargs), + 'project_guids': ArrayAgg('family__project__guid', distinct=True), + 'family_guids': ArrayAgg('family__guid', **family_array_kwargs), } if skip_individual_guid: annotations['num_unaffected'] = Count( - 'individual_id', distinct=True, filter=Q(individual__affected=Individual.AFFECTED_STATUS_UNAFFECTED), + 'id', distinct=True, filter=Q(affected=Individual.AFFECTED_STATUS_UNAFFECTED), ) if annotate_affected_males: - annotations['affected_male_family_guids'] = ArrayAgg('individual__family__guid', distinct=True, filter=Q( - individual__affected=Individual.AFFECTED_STATUS_AFFECTED, individual__sex__in=Individual.MALE_SEXES, + annotations['affected_male_family_guids'] = ArrayAgg('family__guid', distinct=True, filter=Q( + affected=Individual.AFFECTED_STATUS_AFFECTED, sex__in=Individual.MALE_SEXES, )) else: annotations['samples'] = ArrayAgg(JSONObject( - affected='individual__affected', sex='individual__sex', sample_id='sample_id', sample_type='sample_type', - family_guid=F('individual__family__guid'), individual_guid=F('individual__guid'), + affected='affected', sex='sex', sample_id='individual_id', sample_type='active_datasets__sample_type', + family_guid=F('family__guid'), individual_guid=F('guid'), )) - return samples.aggregate(**annotations) + return individuals.aggregate(**annotations) def _get_sample_data(families, dataset_type, annotate_affected_males=False, allow_no_samples=False, inheritance_mode=None, inheritance_filter=None): sample_type = None if dataset_type.startswith(Dataset.DATASET_TYPE_SV_CALLS): dataset_type, sample_type = dataset_type.split('_') - samples = _get_valid_samples(families, dataset_type, sample_type, allow_no_samples) - if not samples: + individuals = _get_valid_samples(families, dataset_type, sample_type, allow_no_samples) + if not individuals: return {} individual_affected_status = (inheritance_filter or {}).get('affected') affected_family_only = inheritance_mode and not individual_affected_status - sample_data = _get_sample_metadata(samples, affected_family_only, annotate_affected_males) + sample_data = _get_sample_metadata(individuals, affected_family_only, annotate_affected_males) family_guids = set(sample_data.pop('family_guids')) sample_data['num_families'] = len(family_guids) @@ -574,8 +574,8 @@ def _get_sample_data(families, dataset_type, annotate_affected_males=False, allo sample_data['sample_type_families'][sample['sample_type']].add(sample['family_guid']) else: sample_data['sample_type_families'] = dict( - samples.filter(individual__family__guid__in=family_guids).values('sample_type').values_list( - 'sample_type', ArrayAgg('individual__family__guid', distinct=True), + individuals.filter(family__guid__in=family_guids).values('active_datasets__sample_type').values_list( + 'active_datasets__sample_type', ArrayAgg('family__guid', distinct=True), ) ) if len(sample_data['sample_type_families']) == 2: @@ -588,12 +588,12 @@ def _get_sample_data(families, dataset_type, annotate_affected_males=False, allo if set(families) - multi_families } sample_data['sample_type_families']['multi'] = multi_families - _add_missing_multi_type_samples(samples, sample_data) + _add_missing_multi_type_samples(individuals, sample_data) return sample_data -def _add_missing_multi_type_samples(samples, data): +def _add_missing_multi_type_samples(individuals, data): data['family_missing_type_samples'] = defaultdict(lambda: defaultdict(list)) if 'samples' in data: individual_sample_types = defaultdict(list) @@ -604,9 +604,9 @@ def _add_missing_multi_type_samples(samples, data): for samples in individual_sample_types.values() if len(samples) == 1 ] else: - individual_samples = samples.filter(individual__family__guid__in=data['sample_type_families']['multi'], - ).values('individual_id', family_guid=F('individual__family__guid')).annotate( - samples=ArrayAgg(JSONObject(sample_id='sample_id', sample_type='sample_type')) + individual_samples = individuals.filter(family__guid__in=data['sample_type_families']['multi'], + ).values('id', family_guid=F('individual__family__guid')).annotate( + samples=ArrayAgg(JSONObject(sample_id='individual_id', sample_type='active_datasets__sample_type')) ).filter(samples__len=1) for agg in individual_samples: sample = agg['samples'][0] diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 8d40ca0198..3c30e4f7a3 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -401,7 +401,7 @@ def test_both_sample_types_search(self): Sample.objects.filter(dataset_type='MITO').update(is_active=False) # One family (F000011_11) in a multi-project search has identical exome and genome data. - self._add_sample_type_samples('WES', individual__family__guid='F000011_11') + self._add_sample_type_samples('WES', active_individuals__family__guid='F000011_11') self._assert_expected_search( MULTI_PROJECT_BOTH_SAMPLE_TYPE_VARIANTS, gene_counts=GENE_COUNTS, locus={'rawItems': 'chr1:1-100000000'}, @@ -462,12 +462,14 @@ def test_both_sample_types_search(self): @staticmethod def _add_sample_type_samples(sample_type, dataset_type=None, **sample_filter): - for sample in Sample.objects.filter(**sample_filter): - sample.pk = None - sample.sample_type = sample_type + for dataset in Dataset.objects.filter(**sample_filter): + individuals = dataset.active_individuals.all() + dataset.pk = None + dataset.sample_type = sample_type if dataset_type: - sample.dataset_type = dataset_type - sample.save() + dataset.dataset_type = dataset_type + dataset.save() + dataset.active_individuals.set(individuals) def test_inheritance_filter(self): inheritance_mode = 'any_affected' @@ -826,7 +828,7 @@ def test_location_search(self): ], ) - self._add_sample_type_samples('WES', individual__family__guid='F000014_14') + self._add_sample_type_samples('WES', active_individuals__family__guid='F000014_14') self._assert_expected_search( [SV_VARIANT1, SV_VARIANT2, MULTI_PROJECT_GCNV_VARIANT3, GCNV_VARIANT4], locus=sv_locus, project_families=[*SINGLE_FAMILY_PROJECT_FAMILIES, *SV_PROJECT_FAMILIES], @@ -1829,7 +1831,7 @@ def test_secondary_annotations_filter(self): ], ) - self._add_sample_type_samples('WES', individual__family__guid='F000014_14') + self._add_sample_type_samples('WES', active_individuals__family__guid='F000014_14') self._assert_expected_search( [MULTI_DATA_TYPE_COMP_HET_VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], MULTI_PROJECT_GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]], inheritance_mode='recessive', From 10990f7c186ce2063d845023516d932a7118aa38 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 7 Apr 2026 17:09:53 -0400 Subject: [PATCH 014/119] search test lean up --- clickhouse_search/search_tests.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 3c30e4f7a3..d4031cd365 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -38,7 +38,7 @@ DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, \ format_cached_variant from reference_data.models import Omim -from seqr.models import Project, Family, Sample, Dataset, VariantSearch, VariantSearchResults, SavedVariant, Individual +from seqr.models import Project, Family, Dataset, VariantSearch, VariantSearchResults, SavedVariant, Individual from seqr.views.apis.data_manager_api import trigger_delete_project from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, \ SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \ @@ -242,8 +242,6 @@ def _get_cached_variant(cls, variant, cached_variant_fields): def _set_grch37_search(self): Project.objects.filter(id=1).update(genome_version='37') - Sample.objects.filter(sample_id='HG00732').update(is_active=False) - Sample.objects.exclude(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS).update(is_active=False) def test_single_family_search(self): variant_gene_counts = { @@ -318,7 +316,7 @@ def test_single_project_search(self): gene_counts={**variant_gene_counts, **GCNV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}} ) - self._add_sample_type_samples('WES', dataset_type='SV', guid__in=['S000135_na20870']) + self._add_sample_type_samples('WES', dataset_type='SV', guid__in=['S000129_na19675']) self._assert_expected_search( [GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], gene_counts={ 'ENSG00000129562': {'total': 1, 'families': {'F000002_2': 1, 'F000003_3': 1}}, @@ -398,7 +396,7 @@ def test_all_project_search(self): ) def test_both_sample_types_search(self): - Sample.objects.filter(dataset_type='MITO').update(is_active=False) + Dataset.objects.get(guid='S000149_hg00733').active_individuals.clear() # One family (F000011_11) in a multi-project search has identical exome and genome data. self._add_sample_type_samples('WES', active_individuals__family__guid='F000011_11') @@ -637,10 +635,12 @@ def test_inheritance_filter(self): ) # Test deletion in trans with hom alt snp/indel - for sample in Sample.objects.filter(individual__family_id=14): - sample.pk = None - sample.dataset_type = 'SNV_INDEL' - sample.save() + dataset = Dataset.objects.get(guid='S000147_na21234') + dataset_individuals = dataset.active_individuals.all() + dataset.pk = None + dataset.dataset_type = 'SNV_INDEL' + dataset.save() + dataset.active_individuals.set(dataset_individuals) self._assert_expected_search( [[SV_VARIANT1, SV_VARIANT2], [SV_VARIANT1, PROJECT_4_COMP_HET_VARIANT], PROJECT_4_COMP_HET_VARIANT, SV_VARIANT4], inheritance_mode=inheritance_mode, inheritance_filter=sv_affected, **COMP_HET_ALL_PASS_FILTERS, @@ -948,7 +948,7 @@ def test_invalid_search(self): self._assert_expected_search_error('Location must be specified to search across multiple projects', project_families=MULTI_PROJECT_PROJECT_FAMILIES) - Sample.objects.filter(guid='S000143_na20885').update(sample_id='HG00732') + Individual.objects.filter(guid='I000015_na20885').update(individual_id='HG00732') self._assert_expected_search_error( 'The following samples are incorrectly configured and have different affected statuses in different projects: ' 'HG00732 (1kg project nåme with uniçøde/ Test Reprocessed Project)', @@ -1855,7 +1855,6 @@ def test_secondary_annotations_filter(self): 'familyGuids': ['F000002_2_x'], 'genotypes': {k: {**v, 'familyGuid': 'F000002_2_x'} for k, v in MULTI_DATA_TYPE_COMP_HET_VARIANT2['genotypes'].items()} } - Sample.objects.filter(guid='S000146_hg00732').update(is_active=False) Family.objects.filter(guid='F000002_2').update(guid='F000002_2_x') self._assert_expected_search( [[missing_gt_comp_het_variant, missing_gt_gcnv_variant]], From df6d16ce301ca109e5596b01e838874b8d866c30 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 7 Apr 2026 17:12:59 -0400 Subject: [PATCH 015/119] fix sample filtering --- seqr/views/apis/variant_search_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 1728375f58..04a70413b0 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -103,8 +103,8 @@ def _get_or_create_results_model(search_hash, search_context, user): if search_context.get('unsolvedFamiliesOnly'): families = families.exclude(analysis_status__in=Family.SOLVED_ANALYSIS_STATUSES) if search_context.get('trioFamiliesOnly'): - families = families.annotate(search_sample_count=Count('individual__sample__id', filter=Q( - individual__sample__is_active=True, individual__sample__dataset_type=Dataset.DATASET_TYPE_VARIANT_CALLS, + families = families.annotate(search_sample_count=Count('individual__id', filter=Q( + individual__active_datasets__dataset_type=Dataset.DATASET_TYPE_VARIANT_CALLS, ))).filter( search_sample_count__gte=3, individual__mother__isnull=False, individual__father__isnull=False, ).distinct() From db0a96209e3cc62040122d1a1279918c688b458c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 7 Apr 2026 17:16:38 -0400 Subject: [PATCH 016/119] update sample counts --- seqr/views/utils/variant_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index 37ce9975f2..2186fc7449 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -12,7 +12,7 @@ from matchmaker.models import MatchmakerSubmissionGenes, MatchmakerSubmission from reference_data.models import TranscriptInfo, Omim, GENOME_VERSION_GRCh38 from seqr.models import SavedVariant, VariantSearchResults, Family, LocusList, LocusListInterval, LocusListGene, \ - RnaSeqTpm, PhenotypePrioritization, Project, Sample, Dataset, RnaSample, VariantTag, VariantTagType + RnaSeqTpm, PhenotypePrioritization, Project, Dataset, RnaSample, VariantTag, VariantTagType from seqr.utils.search.utils import variant_dataset_type from seqr.utils.gene_utils import get_genes_for_variants from seqr.views.utils.json_to_orm_utils import create_model_from_json @@ -455,9 +455,9 @@ def _set_response_gene_scores(response, family_genes, gene_ids): def _add_sample_count_stats(response, genome_versions): - sample_counts = Sample.objects.filter( - is_active=True, individual__family__project__is_demo=False, individual__family__project__genome_version__in=genome_versions, - ).values('sample_type', 'dataset_type').annotate(count=Count('*')) + sample_counts = Dataset.objects.filter( + active_individuals__family__project__is_demo=False, active_individuals__family__project__genome_version__in=genome_versions, + ).values('sample_type', 'dataset_type').annotate(count=Count('active_individuals', distinct=True)) counts_by_dataset_type = defaultdict(dict) for sample_type, dataset_type, count in sample_counts.values_list('sample_type', 'dataset_type', 'count'): counts_by_dataset_type[dataset_type][sample_type] = count From 23072da8a51182f12a9af11318616a37e3e9c2fe Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 7 Apr 2026 17:27:36 -0400 Subject: [PATCH 017/119] fix search context --- seqr/views/apis/variant_search_api.py | 14 +++++++------- seqr/views/utils/orm_to_json_utils.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 04a70413b0..c5d23b71c0 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -13,7 +13,7 @@ import re from reference_data.models import GENOME_VERSION_GRCh38 -from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Sample, Dataset +from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Dataset from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, \ variant_lookup, parse_variant_id, export_variants from seqr.utils.search.utils import InvalidSearchException @@ -361,16 +361,16 @@ def search_context_handler(request): projectGuid=Value(project_guid) if project_guid else F('project__guid'), familyGuid=F('guid'), analysisStatus=F('analysis_status'), + sampleTypes=ArrayAgg( + JSONObject(sampleType='individual__active_datasets__sample_type', datasetType='individual__active_datasets__dataset_type', isActive=Value(True)), + distinct=True, filter=Q(individual__active_datasets__dataset_type__isnull=False), + ), **FAMILY_ADDITIONAL_VALUES, )} - family_sample_types = Sample.objects.filter(individual__family__project__in=projects, is_active=True).values('individual__family__guid').annotate( - samples=ArrayAgg(JSONObject(sampleType='sample_type', datasetType='dataset_type', isActive=Value(True)), distinct=True)) project_dataset_types = defaultdict(set) - for agg in family_sample_types: - family = response['familiesByGuid'][agg['individual__family__guid']] - family['sampleTypes'] = agg['samples'] - project_dataset_types[family['projectGuid']].update([s['datasetType'] for s in agg['samples']]) + for family in response['familiesByGuid'].values(): + project_dataset_types[family['projectGuid']].update([s['datasetType'] for s in family['sampleTypes']]) for project_guid, dataset_types in project_dataset_types.items(): response['projectsByGuid'][project_guid]['datasetTypes'] = list(dataset_types) diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 44a0d4c803..31e4b8bb4b 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -221,7 +221,7 @@ def _get_case_review_fields(model_cls, has_case_review_perm): createdBy=_user_expr('familyanalysedby__created_by'), dataType='familyanalysedby__data_type', lastModifiedDate='familyanalysedby__last_modified_date', - ), filter=Q(familyanalysedby__isnull=False)), + ), filter=Q(familyanalysedby__isnull=False), distinct=True), 'assignedAnalyst': Case( When(assigned_analyst__isnull=False, then=JSONObject( fullName=_full_name_expr('assigned_analyst'), email=F('assigned_analyst__email'), From 2fd42c8e308bfab13066b2038c2db862cea12350 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 10 Apr 2026 11:48:57 -0400 Subject: [PATCH 018/119] fix prioritized qs --- .../commands/tag_seqr_prioritized_variants.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index 51def868ea..17b14a71dc 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -9,7 +9,7 @@ from clickhouse_search.search import get_clickhouse_variants, ENTRY_CLASS_MAP from panelapp.models import PaLocusListGene from reference_data.models import GENOME_VERSION_GRCh38 -from seqr.models import Project, Family, Individual, Sample, Dataset, LocusList +from seqr.models import Project, Family, Individual, Dataset, LocusList from seqr.utils.communication_utils import send_project_notification from seqr.utils.gene_utils import get_genes from clickhouse_search.constants import ANY_AFFECTED, HOMOZYGOUS_RECESSIVE, X_LINKED_RECESSIVE_MALE_AFFECTED, DE_NOVO, COMPOUND_HET @@ -473,7 +473,7 @@ def handle(self, *args, **options): updates = {update: set() for update in ['matched_families', 'new_tag_keys', 'update_tag_keys', 'skipped_tag_keys']} search_counts = {} samples_by_dataset_type = {} - sample_qs = Sample.objects.filter(individual__family__project=project, is_active=True) + sample_qs = Individual.objects.filter(family__project=project) for dataset_type, searches in SEARCHES.items(): self._run_dataset_type_searches( dataset_type, searches, sample_qs, updates, search_counts, samples_by_dataset_type, family_guid_map, @@ -509,24 +509,24 @@ def handle(self, *args, **options): @classmethod def _run_dataset_type_searches(cls, dataset_type, searches, sample_qs, updates, search_counts, samples_by_dataset_type, family_guid_map, project, exclude_genes, gene_by_moi): is_sv = dataset_type == Dataset.DATASET_TYPE_SV_CALLS - sample_qs = sample_qs.filter(dataset_type=dataset_type) + sample_qs = sample_qs.filter(active_datasets__dataset_type=dataset_type) if is_sv: sample_qs = sample_qs.exclude( - individual__sv_flags__contains=['outlier_num._calls'], individual__affected=Individual.AFFECTED_STATUS_AFFECTED, + sv_flags__contains=['outlier_num._calls'], affected=Individual.AFFECTED_STATUS_AFFECTED, ) - sample_types = list(sample_qs.values_list('sample_type', flat=True).distinct()) + sample_types = list(sample_qs.values_list('active_datasets__sample_type', flat=True).distinct()) if len(sample_types) > 1: raise CommandError('Variant prioritization not supported for projects with multiple sample types') sample_type = sample_types[0] if is_sv: dataset_type = f'{dataset_type}_{sample_type}' samples_by_family = { - agg['individual__family__guid']: agg for agg in sample_qs.values('individual__family__guid').annotate( + agg['family__guid']: agg for agg in sample_qs.values('family__guid').annotate( affecteds=ArrayAgg( - JSONObject(maternal_guid='individual__mother__guid', paternal_guid='individual__father__guid', sex='individual__sex'), - filter=Q(individual__affected=Individual.AFFECTED_STATUS_AFFECTED), + JSONObject(maternal_guid='mother__guid', paternal_guid='father__guid', sex='sex'), + filter=Q(affected=Individual.AFFECTED_STATUS_AFFECTED), ), - unaffected_guids=ArrayAgg('individual__guid', filter=Q(individual__affected=Individual.AFFECTED_STATUS_UNAFFECTED)), + unaffected_guids=ArrayAgg('guid', filter=Q(affected=Individual.AFFECTED_STATUS_UNAFFECTED)), ).filter(affecteds__len__gt=0) } samples_by_dataset_type[dataset_type] = samples_by_family From cea838c1ef4df145d4481e14350209a30fa8093c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 10 Apr 2026 11:52:57 -0400 Subject: [PATCH 019/119] fix relaod genotypes --- .../management/commands/reload_saved_variant_genotypes.py | 8 ++++---- .../tests/reload_saved_variant_genotypes_tests.py | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/seqr/management/commands/reload_saved_variant_genotypes.py b/seqr/management/commands/reload_saved_variant_genotypes.py index 86b5c20b14..0af32763aa 100644 --- a/seqr/management/commands/reload_saved_variant_genotypes.py +++ b/seqr/management/commands/reload_saved_variant_genotypes.py @@ -1,7 +1,7 @@ import logging from django.core.management.base import BaseCommand from seqr.management.commands.check_for_new_samples_from_pipeline import update_project_saved_variant_genotypes -from seqr.models import Project, Family, Sample, Dataset +from seqr.models import Project, Family, Dataset logger = logging.getLogger(__name__) @@ -18,12 +18,12 @@ def handle(self, *args, **options): family_guid = options['family_guid'] family_guids = [family_guid] if family_guid else Family.objects.filter(project=project).values_list('guid', flat=True) - samples = Sample.objects.filter(individual__family__project_id=project.id, is_active=True) + datasets = Dataset.objects.filter(active_individuals__family__project_id=project.id) if family_guid: - samples = samples.filter(individual__family__guid=family_guid) + datasets = datasets.filter(active_individuals__family__guid=family_guid) dataset_types = { f'{dataset_type}_{sample_type}' if dataset_type == Dataset.DATASET_TYPE_SV_CALLS else dataset_type - for dataset_type, sample_type in samples.values_list('dataset_type', 'sample_type').distinct() + for dataset_type, sample_type in datasets.values_list('dataset_type', 'sample_type').distinct() } for dataset_type in sorted(dataset_types): update_project_saved_variant_genotypes(project, family_guids, dataset_type) diff --git a/seqr/management/tests/reload_saved_variant_genotypes_tests.py b/seqr/management/tests/reload_saved_variant_genotypes_tests.py index 4f7f8cf3e9..50ec2d6003 100644 --- a/seqr/management/tests/reload_saved_variant_genotypes_tests.py +++ b/seqr/management/tests/reload_saved_variant_genotypes_tests.py @@ -1,7 +1,7 @@ from django.core.management import call_command from seqr.views.utils.test_utils import AnvilAuthenticationTestCase -from seqr.models import SavedVariant, Sample +from seqr.models import SavedVariant, Dataset class ReloadSavedVariantGenotypesTest(AnvilAuthenticationTestCase): @@ -9,7 +9,8 @@ class ReloadSavedVariantGenotypesTest(AnvilAuthenticationTestCase): def test_command(self): # Update fixture data - Sample.objects.filter(id__in=[143, 149]).update(individual_id=18, sample_id='NA21234') + for dataset in Dataset.objects.filter(id__in=[143, 149]): + dataset.active_individuals.set([18]) call_command('reload_saved_variant_genotypes', 'R0004_non_analyst_project') self.assert_json_logs(user=None, expected=[ From eb50bb104573fc240474fb401d4b2274f5c982f1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 10 Apr 2026 11:56:02 -0400 Subject: [PATCH 020/119] codacy clean up --- .../commands/transfer_families_to_different_project.py | 1 - seqr/utils/search/add_data_utils.py | 2 +- seqr/views/apis/dashboard_api.py | 2 +- seqr/views/apis/family_api_tests.py | 1 - 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index 8dea5c4a11..837c12d8a6 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -35,7 +35,6 @@ def handle(self, *args, **options): ] logger.info(f'Skipping {num_found - len(families)} families with analysis groups in the project: {", ".join(group_families)}') - family_guids = list(families.values_list('guid', flat=True)) trigger_delete_families_search(from_project, list(families.values_list('guid', flat=True))) remaining_families = Family.objects.filter(project=from_project).exclude(id__in={f.id for f in families}) diff --git a/seqr/utils/search/add_data_utils.py b/seqr/utils/search/add_data_utils.py index 43881c22b7..2feefa04fa 100644 --- a/seqr/utils/search/add_data_utils.py +++ b/seqr/utils/search/add_data_utils.py @@ -5,7 +5,7 @@ import requests from reference_data.models import GeneInfo, GENOME_VERSION_LOOKUP -from seqr.models import Sample, Dataset, Individual, Project +from seqr.models import Dataset, Individual, Project from seqr.utils.communication_utils import send_project_notification, safe_post_to_slack from seqr.utils.file_utils import does_file_exist from seqr.utils.logging_utils import SeqrLogger diff --git a/seqr/views/apis/dashboard_api.py b/seqr/views/apis/dashboard_api.py index f4a0ceb6bd..d11a4d9bc1 100644 --- a/seqr/views/apis/dashboard_api.py +++ b/seqr/views/apis/dashboard_api.py @@ -4,7 +4,7 @@ from django.db import models from django.db.models.functions import Coalesce -from seqr.models import ProjectCategory, Individual, Dataset, RnaSample, Family, Project +from seqr.models import ProjectCategory, Individual, RnaSample, Family, Project from seqr.views.utils.individual_utils import check_project_individuals_deletable from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.orm_to_json_utils import get_json_for_projects diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index 847d190666..e8fa545231 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import json -import pdb import mock from copy import deepcopy From b2749ef0ee251c66851acc03f15f31265fae5360 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 10 Apr 2026 13:57:32 -0400 Subject: [PATCH 021/119] update new datasets in check samples --- .../check_for_new_samples_from_pipeline.py | 131 ++++++---------- ...eck_for_new_samples_from_pipeline_tests.py | 148 ++++++++++-------- seqr/migrations/0087_dataset.py | 2 +- seqr/models.py | 2 +- 4 files changed, 133 insertions(+), 150 deletions(-) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 518e5fa8c4..6f295209a6 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -10,12 +10,13 @@ from clickhouse_search.search import get_clickhouse_genotypes from reference_data.models import GENOME_VERSION_LOOKUP -from seqr.models import Family, Sample, Dataset, Project, Individual, SavedVariant +from seqr.models import Family, Dataset, Project, Individual, SavedVariant from seqr.utils.communication_utils import safe_post_to_slack, send_project_email from seqr.utils.file_utils import file_iter, list_files, is_google_bucket_file_path from seqr.utils.search.add_data_utils import notify_search_data_loaded, update_airtable_loading_tracking_status from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS from seqr.views.utils.export_utils import write_multiple_files +from seqr.views.utils.json_to_orm_utils import create_model_from_json from seqr.views.utils.permissions_utils import is_internal_anvil_project, project_has_anvil from seqr.views.utils.variant_utils import reset_cached_search_results from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, PIPELINE_DATA_DIR, ANVIL_UI_URL, IS_ANVIL_LOADING_DELAY, \ @@ -76,7 +77,7 @@ def handle(self, *args, **options): def _load_success_runs(self, runs, success_run_dirs): - loaded_runs = set(Sample.objects.filter(data_source__isnull=False).values_list('data_source', flat=True)) + loaded_runs = {source for sources in Dataset.objects.values_list('data_source', flat=True) for source in sources.split(',')} new_runs = { run_dir: run_details for run_dir, run_details in runs.items() if run_dir in success_run_dirs and run_details['run_version'] not in loaded_runs @@ -206,18 +207,29 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers family_project_map = {f.guid: f.project for f in families.select_related('project')} families_by_project = defaultdict(list) samples_by_project = defaultdict(list) + num_samples = 0 for family_guid, sample_ids in metadata['family_samples'].items(): project = family_project_map[family_guid] families_by_project[project].append(family_guid) samples_by_project[project] += sample_ids + num_samples += len(sample_ids) - sample_project_tuples = [] + individuals_by_project = {} + missing_samples = set() + all_individual_ids = set() invalid_genome_version_projects = [] for project, sample_ids in samples_by_project.items(): - sample_project_tuples += [(sample_id, project.id) for sample_id in sample_ids] project_genome_version = GENOME_VERSION_LOOKUP.get(project.genome_version, project.genome_version) if project_genome_version != genome_version: invalid_genome_version_projects.append((project.guid, project_genome_version)) + continue + + matched_individuals = dict( + Individual.objects.filter(family__project=project, individual_id__in=sample_ids).values_list('id', 'individual_id') + ) + missing_samples.update(set(sample_ids) - set(matched_individuals.values())) + individuals_by_project[project] = matched_individuals + all_individual_ids.update(matched_individuals.keys()) if invalid_genome_version_projects: raise CommandError( @@ -226,14 +238,18 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers ) sample_type = metadata['sample_type'] - logger.info(f'Loading {len(sample_project_tuples)} {sample_type} {dataset_type} samples in {len(samples_by_project)} projects') - new_samples = cls._match_and_update_search_samples( - sample_project_tuples, sample_type, dataset_type, data_source=run_version, elasticsearch_index=';'.join(metadata['callsets']), - ) + logger.info(f'Loading {num_samples} {sample_type} {dataset_type} samples in {len(samples_by_project)} projects') + if missing_samples: + sample_ids = ', '.join(sorted(missing_samples)) + raise ValueError(f'Matches not found for sample ids: {sample_ids}') - new_samples_by_project = dict(new_samples.values('individual__family__project').annotate( - samples=ArrayAgg('sample_id', distinct=True), - ).values_list('individual__family__project', 'samples')) + cls._update_matched_families(all_individual_ids, dataset_type, sample_type) + + new_samples_by_project = {} + for project, individuals in individuals_by_project.items(): + new_samples_by_project[project.id] = cls._match_and_update_search_datasets( + individuals, sample_type, dataset_type, data_source=run_version, + ) split_project_pdos = cls._report_loading_success( dataset_type, sample_type, run_version, samples_by_project, new_samples_by_project, @@ -412,72 +428,38 @@ def _update_project_saved_variant_genotypes(project, family_guids, dataset_type) return updates @classmethod - def _match_and_update_search_samples(cls, sample_project_tuples, sample_type, dataset_type, **sample_data): - individual_ids_by_keys = cls._get_matched_individuals(sample_project_tuples) - individual_ids = individual_ids_by_keys.values() - matched_samples = Sample.objects.filter(is_active=True, dataset_type=dataset_type, sample_type=sample_type) - cls._update_matched_families(individual_ids, matched_samples) - - sample_guids_by_individual = dict(Sample.objects.filter( - individual_id__in=individual_ids, sample_type=sample_type, dataset_type=dataset_type, **sample_data, - ).values_list('individual_id', 'guid')) - remaining_key_individuals = { - key: individual_id for key, individual_id in individual_ids_by_keys.items() - if individual_id not in sample_guids_by_individual - } + def _match_and_update_search_datasets(cls, individuals, sample_type, dataset_type, data_source): loaded_date = timezone.now() - created_sample_guids = cls._create_samples( - remaining_key_individuals, loaded_date=loaded_date, sample_type=sample_type, dataset_type=dataset_type, - **sample_data, - ) if remaining_key_individuals else [] - - sample_guids = list(sample_guids_by_individual.values()) + created_sample_guids - - activated_sample_guids = Sample.bulk_update(user=None, update_json={ - 'is_active': True, - 'loaded_date': loaded_date, - **sample_data, - }, guid__in=sample_guids, is_active=False) - - inactivate_samples = matched_samples.filter(individual_id__in=individual_ids).exclude(guid__in=sample_guids) - inactivated_sample_guids = Sample.bulk_update( - user=None, update_json={'is_active': False}, queryset=inactivate_samples, - ) - - previous_loaded_individuals = set(Sample.objects.filter(guid__in=inactivated_sample_guids).values_list('individual_id', flat=True)) - return Sample.objects.filter(guid__in=activated_sample_guids).exclude(individual_id__in=previous_loaded_individuals) + dataset = create_model_from_json(Dataset, { + 'dataset_type': dataset_type, 'sample_type': sample_type, 'data_source': data_source, 'loaded_date': loaded_date, + }, user=None) + dataset.active_individuals.set(individuals.keys()) + + inactivate_datasets = Dataset.objects.filter( + dataset_type=dataset_type, sample_type=sample_type, active_individuals__id__in=individuals.keys(), + ).exclude(id=dataset.id) + loaded_individuals = set() + for dataset in inactivate_datasets: + inactivate_individuals = dataset.active_individuals.filter(id__in=individuals.keys()) + loaded_individuals.update(inactivate_individuals.values_list('id', flat=True)) + dataset.inactive_individuals.add(*inactivate_individuals) + dataset.active_individuals.remove(*inactivate_individuals) + + return [sample_id for individual_id, sample_id in individuals.items() if individual_id not in loaded_individuals] @staticmethod - def _get_matched_individuals(sample_project_tuples): - individual_ids_by_keys = { - (individual_id, project_id): individual_db_id - for individual_db_id, individual_id, project_id in Individual.objects.filter( - family__project_id__in={project_id for _, project_id in sample_project_tuples}, - individual_id__in={sample_id for sample_id, _ in sample_project_tuples}, - ).values_list('id', 'individual_id', 'family__project_id') - if (individual_id, project_id) in sample_project_tuples - } - - missing_keys = set(sample_project_tuples) - set(individual_ids_by_keys.keys()) - if missing_keys: - sample_ids = ', '.join(sorted([sample_id for sample_id, _ in missing_keys])) - raise ValueError(f'Matches not found for sample ids: {sample_ids}') - - return individual_ids_by_keys - - @staticmethod - def _update_matched_families(individual_ids, matched_samples): + def _update_matched_families(individual_ids, dataset_type, sample_type): included_families = dict( Family.objects.filter(individual__id__in=individual_ids).values_list('id', 'analysis_status') ) - missing_individuals = matched_samples.filter( - individual__family_id__in=included_families, - ).exclude(individual_id__in=individual_ids).values( - 'individual__family__family_id', - ).annotate(individual_ids=ArrayAgg('individual__individual_id', ordering='individual__individual_id')) + missing_individuals = Individual.objects.filter( + family_id__in=included_families, active_datasets__dataset_type=dataset_type, active_datasets__sample_type=sample_type, + ).exclude(id__in=individual_ids).values( + 'family__family_id', + ).annotate(individual_ids=ArrayAgg('individual_id', ordering='individual_id')) if missing_individuals: missing_summary = ', '.join(sorted([ - f"{agg['individual__family__family_id']} ({', '.join(agg['individual_ids'])})" for agg in missing_individuals + f"{agg['family__family_id']} ({', '.join(agg['individual_ids'])})" for agg in missing_individuals ])) raise ValueError( f'The following families are included in the callset but are missing some family members: {missing_summary}' @@ -491,17 +473,6 @@ def _update_matched_families(individual_ids, matched_samples): user=None, update_json={'analysis_status': Family.ANALYSIS_STATUS_ANALYSIS_IN_PROGRESS}, id__in=family_ids_to_update, ) - @staticmethod - def _create_samples(remaining_key_individuals, **sample_params): - new_samples = [ - Sample( - individual_id=individual_id, - sample_id=sample_key[0], - **sample_params, - ) for sample_key, individual_id in remaining_key_individuals.items()] - new_sample_models = Sample.bulk_create(user=None, new_models=new_samples) - return [s.guid for s in new_sample_models] - update_individuals_sample_qc = Command._update_individuals_sample_qc get_pipeline_runs = Command._get_runs diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 55e19a2553..6967ec56d1 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -6,27 +6,20 @@ import mock import responses -from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, AuthenticationTestCase, DifferentDbTransactionSupportMixin -from seqr.models import Project, Family, Individual, Sample, SavedVariant +from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, AuthenticationTestCase +from seqr.models import Project, Family, Individual, Dataset, SavedVariant SEQR_URL = 'https://seqr.broadinstitute.org/' PROJECT_GUID = 'R0003_test' EXTERNAL_PROJECT_GUID = 'R0004_non_analyst_project' -GUID_ID = 54321 -GCNV_GUID_ID = 12345 -NEW_SAMPLE_GUID_P3 = f'S00000{GUID_ID}_na20888' -NEW_SAMPLE_GUID_P4 = f'S00000{GUID_ID}_na21234' -REPLACED_SAMPLE_GUID = f'S00000{GUID_ID}_na20885' -EXISTING_INACTIVE_SAMPLE_GUID = 'S000154_na20889' -ACTIVE_SAMPLE_GUID = f'S00000{GUID_ID}_na20889' -EXISTING_WGS_SAMPLE_GUID = 'S000144_na20888' -EXISTING_SV_SAMPLE_GUID = 'S000147_na21234' -SAMPLE_GUIDS = [ACTIVE_SAMPLE_GUID, REPLACED_SAMPLE_GUID, NEW_SAMPLE_GUID_P3, NEW_SAMPLE_GUID_P4] -GCNV_SAMPLE_GUID = f'S00000{GCNV_GUID_ID}_na20889' -EXISTING_GCNV_SAMPLE_GUIDS = ['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733'] -GCNV_SAMPLE_GUIDS = [f'S00000{GCNV_GUID_ID}_hg00731', f'S00000{GCNV_GUID_ID}_hg00732', f'S00000{GCNV_GUID_ID}_hg00733', GCNV_SAMPLE_GUID] -OLD_DATA_SAMPLE_GUID = 'S000143_na20885' +EXISTING_INACTIVE_DATASET_GUID = 'S000154_na20889' +EXISTING_WGS_DATASET_GUID = 'S000144_na20888' +EXISTING_SV_DATASET_GUID = 'S000147_na21234' +EXISTING_GCNV_DATASET_GUID = 'S000145_hg00731' +OLD_DATA_DATASET_GUID = 'S000143_na20885' +NEW_DATASET_GUIDS = ['D0000155_snv_indel_wes_2025_09', 'D0000156_snv_indel_wes_2025_09'] +NEW_GCNV_DATASET_GUIDS = ['D0000157_sv_wes_2025_09_23_000', 'D0000158_sv_wes_2025_09_23_000'] namespace_path = 'ext-data/anvil-non-analyst-project 1000 Genomes Demo' anvil_link = f'{namespace_path}' @@ -384,14 +377,14 @@ def set_up(self): self.mock_redis = patcher.start() self.mock_redis.return_value.keys.side_effect = lambda pattern: [pattern] self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.models.random.randint') - mock_rand_int = patcher.start() - mock_rand_int.side_effect = [GUID_ID, GUID_ID, GUID_ID, GUID_ID, GCNV_GUID_ID, GCNV_GUID_ID, GCNV_GUID_ID, GCNV_GUID_ID, GUID_ID, GUID_ID, GUID_ID, GUID_ID] - self.addCleanup(patcher.stop) patcher = mock.patch('seqr.management.commands.check_for_new_samples_from_pipeline.PIPELINE_DATA_DIR') mock_data_dir = patcher.start() mock_data_dir.__str__.return_value = self.MOCK_DATA_DIR self.addCleanup(patcher.stop) + patcher = mock.patch('seqr.management.commands.check_for_new_samples_from_pipeline.timezone.now') + mock_now = patcher.start() + mock_now.return_value = datetime(2025, 9, 23) + self.addCleanup(patcher.stop) patcher = mock.patch('seqr.views.utils.export_utils.TemporaryDirectory') mock_temp_dir = patcher.start() mock_temp_dir.return_value.__enter__.return_value = '/mock/tmp' @@ -421,7 +414,7 @@ def set_up(self): patcher.start() self.addCleanup(patcher.stop) - Sample.objects.filter(guid=OLD_DATA_SAMPLE_GUID).update(sample_type='WES') + Dataset.objects.filter(guid=OLD_DATA_DATASET_GUID).update(sample_type='WES') def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=6): self._set_loading_files() @@ -463,16 +456,16 @@ def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=6): num_calls = self._assert_expected_airtable_calls(bool(run_loading_logs), single_call) self.assertEqual(len(responses.calls), num_calls) - def _test_success_call(self, anvil_email_calls): + def _test_success_call(self, anvil_email_calls, next_dataset_id=155): Project.objects.filter(id__in=[1, 3]).update(genome_version=38) + self.maxDiff = None self._test_call(run_loading_logs={ 'GRCh38/SNV_INDEL': [ ('Loading 4 WES SNV_INDEL samples in 2 projects', None), ('update 2 Familys', {'dbUpdate': mock.ANY}), - ('create 4 Samples', {'dbUpdate': mock.ANY}), - ('update 4 Samples', {'dbUpdate': mock.ANY}), - ('update 1 Samples', {'dbUpdate': mock.ANY}), + (f'create Dataset D0000{next_dataset_id}_snv_indel_wes_2025_09', {'dbUpdate': mock.ANY}), + (f'create Dataset D0000{next_dataset_id+1}_snv_indel_wes_2025_09', {'dbUpdate': mock.ANY}), ] + self.AIRTABLE_LOGS + [ ('update 3 Familys', {'dbUpdate': mock.ANY}), ] + self.UPDATE_SAMPLE_LOGS, @@ -482,9 +475,8 @@ def _test_success_call(self, anvil_email_calls): 'GRCh38/SV': [ ('Loading 4 WES SV samples in 2 projects', None), ('update 1 Familys', {'dbUpdate': mock.ANY}), - ('create 4 Samples', {'dbUpdate': mock.ANY}), - ('update 4 Samples', {'dbUpdate': mock.ANY}), - ('update 3 Samples', {'dbUpdate': mock.ANY}), + (f'create Dataset D0000{next_dataset_id+2}_sv_wes_2025_09_23_000', {'dbUpdate': mock.ANY}), + (f'create Dataset D0000{next_dataset_id+3}_sv_wes_2025_09_23_000', {'dbUpdate': mock.ANY}), ('Reloading saved variants in 2 projects', None), ('Updated 0 variants in 1 families for project 1kg project nåme with uniçøde', None), ('Updated 0 variants in 1 families for project Test Reprocessed Project', None), @@ -561,11 +553,15 @@ def _test_success_call(self, anvil_email_calls): self.assertDictEqual(self.mock_email.return_value.merge_data, {}) self.assertEqual(self.manager_user.notifications.count(), 5) - self.assertEqual( - str(self.manager_user.notifications.first()), 'Test Reprocessed Project Loaded 1 new WES SV samples 0 minutes ago') + manager_notification = self.manager_user.notifications.first() + self.assertEqual(manager_notification.actor.name, 'Test Reprocessed Project') + self.assertEqual(manager_notification.verb, 'Loaded 1 new WES SV samples') + self.assertEqual(manager_notification.timestamp.isoformat(), '2025-09-23T00:00:00+00:00') self.assertEqual(self.collaborator_user.notifications.count(), 2) - self.assertEqual( - str(self.collaborator_user.notifications.first()), 'Non-Analyst Project Loaded 1 new WES samples 0 minutes ago') + collaborator_notification = self.collaborator_user.notifications.first() + self.assertEqual(collaborator_notification.actor.name, 'Non-Analyst Project') + self.assertEqual(collaborator_notification.verb, 'Loaded 1 new WES samples') + self.assertEqual(collaborator_notification.timestamp.isoformat(), '2025-09-23T00:00:00+00:00') def _additional_loading_logs(self, data_type, version): return [] @@ -596,7 +592,7 @@ def test_command(self): 'auto__2024-09-14': 'Data has genome version GRCh38 but the following projects have conflicting versions: R0001_1kg (GRCh37), R0003_test (GRCh37)', } self._test_call(error_logs=error_logs) - self.assertEqual(Sample.objects.filter(guid__in=SAMPLE_GUIDS + GCNV_SAMPLE_GUIDS).count(), 0) + self.assertEqual(Dataset.objects.filter(guid__in=NEW_DATASET_GUIDS + NEW_GCNV_DATASET_GUIDS).count(), 0) # Update fixture data to allow testing edge cases svs = SavedVariant.objects.filter(guid__in=['SV0000002_1248367227_r0390_100', 'SV0000006_1248367227_r0003_tes', 'SV0000007_prefix_19107_DEL_r00']) @@ -610,55 +606,71 @@ def test_command(self): self._test_success_call(self._anvil_email_calls()) # Tests Sample models created/updated - snv_indel_samples = Sample.objects.filter(data_source='auto__2023-08-09') - gcnv_samples = Sample.objects.filter(data_source='auto__2024-09-14') - updated_sample_models = snv_indel_samples | gcnv_samples - self.assertSetEqual({'WES'}, set(updated_sample_models.values_list('sample_type', flat=True))) - self.assertSetEqual({True}, set(updated_sample_models.values_list('is_active', flat=True))) + snv_indel_datasets = Dataset.objects.filter(data_source='auto__2023-08-09') + gcnv_datasets = Dataset.objects.filter(data_source='auto__2024-09-14') + new_dataset_models = snv_indel_datasets | gcnv_datasets + self.assertSetEqual({'WES'}, set(new_dataset_models.values_list('sample_type', flat=True))) self.assertSetEqual( - {datetime.now().strftime('%Y-%m-%d')}, - {date.strftime('%Y-%m-%d') for date in updated_sample_models.values_list('loaded_date', flat=True)} + {'2025-09-23'}, + {date.strftime('%Y-%m-%d') for date in new_dataset_models.values_list('loaded_date', flat=True)} ) - self.assertSetEqual(set(snv_indel_samples.values_list('guid', flat=True)), set(SAMPLE_GUIDS)) - self.assertSetEqual({'SNV_INDEL'}, set(snv_indel_samples.values_list('dataset_type', flat=True))) - self.assertSetEqual({'1kg.vcf.gz;new_samples.vcf.gz'}, set(snv_indel_samples.values_list('elasticsearch_index', flat=True))) + self.assertSetEqual(set(snv_indel_datasets.values_list('guid', flat=True)), set(NEW_DATASET_GUIDS)) + self.assertSetEqual({'SNV_INDEL'}, set(snv_indel_datasets.values_list('dataset_type', flat=True))) - self.assertSetEqual(set(gcnv_samples.values_list('guid', flat=True)), set(GCNV_SAMPLE_GUIDS)) - self.assertSetEqual({'SV'}, set(gcnv_samples.values_list('dataset_type', flat=True))) - self.assertSetEqual({'gcnv.bed.gz'}, set(gcnv_samples.values_list('elasticsearch_index', flat=True))) + self.assertSetEqual(set(gcnv_datasets.values_list('guid', flat=True)), set(NEW_GCNV_DATASET_GUIDS)) + self.assertSetEqual({'SV'}, set(gcnv_datasets.values_list('dataset_type', flat=True))) - self.assertFalse(Sample.objects.get(guid=OLD_DATA_SAMPLE_GUID).is_active) + old_dataset = Dataset.objects.get(guid=OLD_DATA_DATASET_GUID) + self.assertListEqual(list(old_dataset.active_individuals.values_list('id', flat=True)), []) + self.assertListEqual(list(old_dataset.inactive_individuals.values_list('id', flat=True)), [15]) - previous_gcnv_samples = Sample.objects.filter(guid__in=EXISTING_GCNV_SAMPLE_GUIDS) - self.assertEqual(len(previous_gcnv_samples), len(EXISTING_GCNV_SAMPLE_GUIDS)) - self.assertFalse(any(previous_gcnv_samples.values_list('is_active', flat=True))) + old_gcnv_dataset = Dataset.objects.get(guid=EXISTING_GCNV_DATASET_GUID) + self.assertListEqual(list(old_gcnv_dataset.active_individuals.values_list('id', flat=True)), []) + self.assertListEqual(list(old_gcnv_dataset.inactive_individuals.values_list('id', flat=True)), [4, 5, 6]) # Previously loaded WGS data should be unchanged by loading WES data self.assertEqual( - Sample.objects.get(guid=EXISTING_WGS_SAMPLE_GUID).last_modified_date.strftime('%Y-%m-%d'), '2017-03-13') + Dataset.objects.get(guid=EXISTING_WGS_DATASET_GUID).last_modified_date.strftime('%Y-%m-%d'), '2017-03-13') # Previously loaded SV data should be unchanged by loading SNV_INDEL data - sv_sample = Sample.objects.get(guid=EXISTING_SV_SAMPLE_GUID) - self.assertEqual(sv_sample.last_modified_date.strftime('%Y-%m-%d'), '2018-03-13') - self.assertTrue(sv_sample.is_active) + sv_dataset= Dataset.objects.get(guid=EXISTING_SV_DATASET_GUID) + self.assertEqual(sv_dataset.last_modified_date.strftime('%Y-%m-%d'), '2018-03-13') + self.assertListEqual(list(sv_dataset.active_individuals.values_list('id', flat=True)), [18, 19, 21]) + self.assertListEqual(list(sv_dataset.inactive_individuals.values_list('id', flat=True)), []) # Test Individual models properly associated with Samples self.assertSetEqual( - set(Individual.objects.get(guid='I000015_na20885').sample_set.values_list('guid', flat=True)), - {REPLACED_SAMPLE_GUID, OLD_DATA_SAMPLE_GUID} + set(Individual.objects.get(guid='I000015_na20885').active_datasets.values_list('guid', flat=True)), + {NEW_DATASET_GUIDS[0]} + ) + self.assertSetEqual( + set(Individual.objects.get(guid='I000015_na20885').inactive_datasets.values_list('guid', flat=True)), + {OLD_DATA_DATASET_GUID} + ) + self.assertSetEqual( + set(Individual.objects.get(guid='I000016_na20888').active_datasets.values_list('guid', flat=True)), + {NEW_DATASET_GUIDS[0]} + ) + self.assertSetEqual( + set(Individual.objects.get(guid='I000016_na20888').inactive_datasets.values_list('guid', flat=True)), + {EXISTING_WGS_DATASET_GUID} + ) + self.assertSetEqual( + set(Individual.objects.get(guid='I000017_na20889').active_datasets.values_list('guid', flat=True)), + {NEW_DATASET_GUIDS[0], NEW_GCNV_DATASET_GUIDS[1]} ) self.assertSetEqual( - set(Individual.objects.get(guid='I000016_na20888').sample_set.values_list('guid', flat=True)), - {EXISTING_WGS_SAMPLE_GUID, NEW_SAMPLE_GUID_P3} + set(Individual.objects.get(guid='I000017_na20889').inactive_datasets.values_list('guid', flat=True)), + {EXISTING_INACTIVE_DATASET_GUID} ) self.assertSetEqual( - set(Individual.objects.get(guid='I000017_na20889').sample_set.values_list('guid', flat=True)), - {EXISTING_INACTIVE_SAMPLE_GUID, ACTIVE_SAMPLE_GUID, GCNV_SAMPLE_GUID} + set(Individual.objects.get(guid='I000018_na21234').active_datasets.values_list('guid', flat=True)), + {NEW_DATASET_GUIDS[1], EXISTING_SV_DATASET_GUID} ) self.assertSetEqual( - set(Individual.objects.get(guid='I000018_na21234').sample_set.values_list('guid', flat=True)), - {EXISTING_SV_SAMPLE_GUID, NEW_SAMPLE_GUID_P4} + set(Individual.objects.get(guid='I000018_na21234').inactive_datasets.values_list('guid', flat=True)), + set(), ) # Test Individual model properly updated with sample qc results @@ -707,26 +719,26 @@ def test_command(self): self.mock_email.reset_mock() self.mock_send_slack.reset_mock() self.mock_redis.reset_mock() - sample_last_modified = Sample.objects.filter( + dataset_last_modified = Dataset.objects.filter( last_modified_date__isnull=False).values_list('last_modified_date', flat=True).order_by('-last_modified_date')[0] call_command('check_for_new_samples_from_pipeline') self.assert_json_logs(user=None, expected=self.LIST_FILE_LOGS[:1] + [('Data already loaded for all 2 runs', None)]) self.mock_email.assert_not_called() self.mock_send_slack.assert_not_called() - self.assertFalse(Sample.objects.filter(last_modified_date__gt=sample_last_modified).exists()) + self.assertFalse(Dataset.objects.filter(last_modified_date__gt=dataset_last_modified).exists()) self.mock_redis.return_value.delete.assert_not_called() # Test reloading shared annotations is skipped if too many saved variants - snv_indel_samples.delete() + snv_indel_datasets.delete() airtable_logs = self.AIRTABLE_LOGS[:-1] if self.AIRTABLE_LOGS: airtable_logs.append(('Fetched 1 AnVIL Seqr Loading Requests Tracking records from airtable', None)) self._test_call(num_runs=2, run_loading_logs={ 'GRCh38/SNV_INDEL': [ ('Loading 4 WES SNV_INDEL samples in 2 projects', None), - ('create 4 Samples', {'dbUpdate': mock.ANY}), - ('update 4 Samples', {'dbUpdate': mock.ANY}), + ('create Dataset D0000159_snv_indel_wes_2025_09', {'dbUpdate': mock.ANY}), + ('create Dataset D0000160_snv_indel_wes_2025_09', {'dbUpdate': mock.ANY}), ] + airtable_logs + self.UPDATE_SAMPLE_LOGS, }) @@ -1030,4 +1042,4 @@ def test_loading_delay_command(self): self._test_success_call(self._anvil_email_calls( email_text=ANVIL_ERROR_TEXT_EMAIL_TEMPLATE.format(error='\n'+ANVIL_ERROR_DELAY), email_html=ANVIL_ERROR_HTML_EMAIL_TEMPLATE.format(error='
'+ANVIL_ERROR_DELAY), - )) \ No newline at end of file + ), next_dataset_id=161) \ No newline at end of file diff --git a/seqr/migrations/0087_dataset.py b/seqr/migrations/0087_dataset.py index 17d413ebc9..7bc5cb367f 100644 --- a/seqr/migrations/0087_dataset.py +++ b/seqr/migrations/0087_dataset.py @@ -99,7 +99,7 @@ class Migration(migrations.Migration): ('last_modified_date', models.DateTimeField(blank=True, db_index=True, null=True)), ('sample_type', models.CharField(choices=[('WES', 'Exome'), ('WGS', 'Whole Genome')], max_length=10)), ('dataset_type', models.CharField(choices=[('SNV_INDEL', 'Variant Calls'), ('SV', 'SV Calls'), ('MITO', 'Mitochondria calls')], max_length=13)), - ('data_source', models.TextField(null=True)), + ('data_source', models.TextField()), ('loaded_date', models.DateTimeField()), ('active_individuals', models.ManyToManyField(related_name='active_datasets', to='seqr.individual')), ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)), diff --git a/seqr/models.py b/seqr/models.py index 83878af0e7..47551870a0 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -772,7 +772,7 @@ class Dataset(ModelWithGUID): sample_type = models.CharField(max_length=10, choices=SAMPLE_TYPE_CHOICES) dataset_type = models.CharField(max_length=13, choices=DATASET_TYPE_CHOICES) - data_source = models.TextField(null=True) + data_source = models.TextField() loaded_date = models.DateTimeField() def __unicode__(self): From 7ae706a1679434fe1c9b746d9583325c6d20ab1e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 10 Apr 2026 14:09:57 -0400 Subject: [PATCH 022/119] fix seqt saved key --- .../management/commands/set_saved_variant_key.py | 6 +++--- .../management/tests/set_saved_variant_key_tests.py | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py index f743d44346..fa61c61db1 100644 --- a/clickhouse_search/management/commands/set_saved_variant_key.py +++ b/clickhouse_search/management/commands/set_saved_variant_key.py @@ -104,7 +104,7 @@ def _query_missing_variants(cls, variant_ids, variant_fields, genome_version=GEN variant_id__in=variant_ids, family__project__genome_version=genome_version, ) num_missing = missing_variants.count() - missing_with_data_qs = missing_variants.filter(family__individual__sample__is_active=True).distinct() + missing_with_data_qs = missing_variants.filter(family__individual__active_datasets__isnull=False).distinct() missing_with_search_data = missing_with_data_qs.values( 'variant_id', *variant_fields, ).annotate(family_ids=ArrayAgg('family__family_id', distinct=True)).order_by('variant_id') @@ -129,7 +129,7 @@ def _resolve_missing_variants(cls, variant_ids, genome_version): @classmethod def _resolve_reloaded_svs(cls, variant_ids): missing_with_search_data, num_missing = cls._query_missing_variants( - list(variant_ids), ['family__individual__sample__sample_type'], + list(variant_ids), ['family__individual__active_datasets__sample_type'], ) logger.info( f'{num_missing} SV variants have no key, {num_missing - len(missing_with_search_data)} of which have no search data' @@ -140,7 +140,7 @@ def _resolve_reloaded_svs(cls, variant_ids): missing_by_sample_type = defaultdict(list) for variant in missing_with_search_data: variant_id = variant['variant_id'] - sample_type = variant['family__individual__sample__sample_type'] + sample_type = variant['family__individual__active_datasets__sample_type'] missing_by_sample_type[sample_type].append(f"{variant_id} - {'; '.join(variant['family_ids'])}" ) for sample_type, variants in missing_by_sample_type.items(): diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py index e53bdf1a2b..9b3d07f479 100644 --- a/clickhouse_search/management/tests/set_saved_variant_key_tests.py +++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py @@ -1,7 +1,7 @@ from django.core.management import call_command import mock -from seqr.models import Project, Sample, SavedVariant +from seqr.models import Project, Dataset, SavedVariant from seqr.views.utils.test_utils import AnvilAuthenticationTestCase MOCK_GCNV_DATA = [ @@ -18,7 +18,10 @@ class SetSavedVariantKeyTest(AnvilAuthenticationTestCase): @classmethod def setUpTestData(cls): Project.objects.filter(id=3).update(genome_version='38') - Sample.objects.filter(guid='S000154_na20889').update(dataset_type='SV', is_active=True) + dataset = Dataset.objects.get(guid='S000154_na20889') + dataset.dataset_type = 'SV' + dataset.save() + dataset.active_individuals.set({17}) for sv in SavedVariant.objects.filter(key__isnull=False): sv.saved_variant_json = { 'genotypes': sv.genotypes, 'populations': {'gnomad': {'af': 0.01}}, From d1a85ff49a53e1b8edbac9925e06cc3fa8a377a7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 10 Apr 2026 14:22:11 -0400 Subject: [PATCH 023/119] misc fixes --- .../tests/check_for_new_samples_from_pipeline_tests.py | 2 +- seqr/views/apis/data_manager_api.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 6967ec56d1..41b0290c07 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -553,7 +553,7 @@ def _test_success_call(self, anvil_email_calls, next_dataset_id=155): self.assertDictEqual(self.mock_email.return_value.merge_data, {}) self.assertEqual(self.manager_user.notifications.count(), 5) - manager_notification = self.manager_user.notifications.first() + manager_notification = self.manager_user.notifications.order_by('-id').first() self.assertEqual(manager_notification.actor.name, 'Test Reprocessed Project') self.assertEqual(manager_notification.verb, 'Loaded 1 new WES SV samples') self.assertEqual(manager_notification.timestamp.isoformat(), '2025-09-23T00:00:00+00:00') diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 23e7fa80e2..5b2bf4708c 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -222,14 +222,14 @@ def get_loaded_projects(request, genome_version, sample_type, dataset_type): if dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS: exclude_sample_type = Dataset.SAMPLE_TYPE_WES if sample_type == Dataset.SAMPLE_TYPE_WGS else Dataset.SAMPLE_TYPE_WGS # Include projects with either the matched sample type OR with no loaded data - projects = projects.exclude(family__individual__sample__sample_type=exclude_sample_type) + projects = projects.exclude(family__individual__active_datasets__sample_type=exclude_sample_type) else: # All other data types can only be loaded to projects which already have loaded data - projects = projects.filter(family__individual__sample__sample_type=sample_type) + projects = projects.filter(family__individual__active_datasets__sample_type=sample_type) projects = projects.distinct().order_by('name').values('name', projectGuid=F('guid'), dataTypeLastLoaded=Max( - 'family__individual__sample__loaded_date', - filter=Q(family__individual__sample__dataset_type=dataset_type) & Q(family__individual__sample__sample_type=sample_type), + 'family__individual__active_datasets__loaded_date', + filter=Q(family__individual__active_datasets__dataset_type=dataset_type) & Q(family__individual__active_datasets__sample_type=sample_type), )) if project_samples: From 85fc5680e820f7794094c46dd2ce6ef13a42365d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 10 Apr 2026 14:31:56 -0400 Subject: [PATCH 024/119] fix both sample type test --- clickhouse_search/search.py | 2 +- clickhouse_search/search_tests.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 1501738edd..b2648c1d79 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -602,7 +602,7 @@ def _add_missing_multi_type_samples(individuals, data): ] else: individual_samples = individuals.filter(family__guid__in=data['sample_type_families']['multi'], - ).values('id', family_guid=F('individual__family__guid')).annotate( + ).values('id', family_guid=F('family__guid')).annotate( samples=ArrayAgg(JSONObject(sample_id='individual_id', sample_type='active_datasets__sample_type')) ).filter(samples__len=1) for agg in individual_samples: diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index d4031cd365..3acd920479 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -406,7 +406,12 @@ def test_both_sample_types_search(self): project_families=MULTI_PROJECT_PROJECT_FAMILIES, check_login=self.check_collaborator_login, ) - self._add_sample_type_samples('WGS', guid__in=['S000132_hg00731']) + self._add_sample_type_samples('WGS', guid='S000129_na19675') + dataset = Dataset.objects.get(guid='S000129_na19675') + dataset.pk = None + dataset.sample_type = 'WGS' + dataset.save() + dataset.active_individuals.add(4) # Variant 1 is de novo in exome but inherited and homozygous in genome. # Variant 2 is inherited and homozygous in exome and de novo and homozygous in genome, so it fails de-novo inheritance when parental data is missing in genome. @@ -428,7 +433,7 @@ def test_both_sample_types_search(self): inheritance_mode='de_novo', quality_filter=None, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) - self._add_sample_type_samples('WGS', guid__in=['S000133_hg00732', 'S000134_hg00733']) + dataset.active_individuals.add(5, 6) self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT3_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], From bec9a81b1a3867f7bbcf960a9cd403b525424e34 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 10 Apr 2026 14:41:52 -0400 Subject: [PATCH 025/119] test fix --- clickhouse_search/search_tests.py | 2 +- seqr/views/apis/data_manager_api_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 3acd920479..cb0165a085 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -428,6 +428,7 @@ def test_both_sample_types_search(self): inheritance_mode='any_affected', quality_filter={'min_gq': 40, 'min_qs': 20}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) + self.maxDiff = None self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1], inheritance_mode='de_novo', quality_filter=None, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, @@ -450,7 +451,6 @@ def test_both_sample_types_search(self): inheritance_mode='de_novo', quality_filter={'min_gq': 40}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES, ) - self.maxDiff = None self._assert_expected_search( [VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, [{**VARIANT2_BOTH_SAMPLE_TYPES, 'selectedMainTranscriptId': 'ENST00000450625'}, GCNV_VARIANT4], diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index cd6681fd08..d9a2143371 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1298,7 +1298,7 @@ class LocalDataManagerAPITest(AuthenticationTestCase, DataManagerAPITest): PROJECT_OPTION = PROJECT_OPTION WGS_PROJECT_OPTIONS = [EMPTY_PROJECT_OPTION] WES_PROJECT_OPTIONS = [ - {'name': '1kg project nåme with uniçøde', 'projectGuid': 'R0001_1kg', 'dataTypeLastLoaded': '2017-02-05T06:25:55.397Z'}, + {'name': '1kg project nåme with uniçøde', 'projectGuid': 'R0001_1kg', 'dataTypeLastLoaded': '2017-02-05T06:13:55.397Z'}, EMPTY_PROJECT_OPTION, ] PROJECT_OPTIONS = [{'projectGuid': 'R0001_1kg'}, PROJECT_OPTION] From 5391aa579948304b12b6310bce4853fca318e6c1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 10 Apr 2026 16:22:49 -0400 Subject: [PATCH 026/119] use datasets on project page --- seqr/views/apis/project_api.py | 38 ++++++++----------- seqr/views/utils/orm_to_json_utils.py | 9 +++++ .../Project/components/ProjectOverview.jsx | 13 +++++-- ui/pages/Project/selectors.js | 28 +++++++++++++- ui/redux/rootReducer.js | 2 +- ui/redux/selectors.js | 37 +++++++++++------- 6 files changed, 84 insertions(+), 43 deletions(-) diff --git a/seqr/views/apis/project_api.py b/seqr/views/apis/project_api.py index 31cb7dcd92..eca9d9d0d3 100644 --- a/seqr/views/apis/project_api.py +++ b/seqr/views/apis/project_api.py @@ -12,7 +12,7 @@ from notifications.models import Notification from matchmaker.models import MatchmakerSubmission -from seqr.models import Project, Family, Individual, Sample, RnaSample, FamilyNote, PhenotypePrioritization, CAN_EDIT +from seqr.models import Project, Family, Individual, Dataset, RnaSample, FamilyNote, PhenotypePrioritization, CAN_EDIT from seqr.utils.file_utils import file_iter from seqr.utils.logging_utils import SeqrLogger from seqr.views.utils.airtable_utils import AirtableSession, ANVIL_REQUEST_TRACKING_TABLE @@ -21,7 +21,7 @@ from seqr.views.utils.individual_utils import delete_individuals from seqr.views.utils.json_utils import create_json_response, _to_snake_case, _to_camel_case from seqr.views.utils.json_to_orm_utils import update_project_from_json, create_model_from_json, update_model_from_json -from seqr.views.utils.orm_to_json_utils import _get_json_for_project, get_json_for_samples, \ +from seqr.views.utils.orm_to_json_utils import _get_json_for_project, get_json_for_samples, get_json_for_datasets, \ get_json_for_project_collaborator_list, get_json_for_matchmaker_submissions, \ get_json_for_family_notes, _get_json_for_individuals, get_json_for_project_collaborator_groups, \ FAMILY_ADDITIONAL_VALUES @@ -265,25 +265,24 @@ def project_families(request, project_guid): def project_overview(request, project_guid): project = get_project_and_check_permissions(project_guid, request.user) - sample_load_counts, sample_models = _sample_load_counts( - Sample, project, 'sample_type', 'dataset_type', loadedDate=TruncDate('loaded_date'), - ) - rna_sample_load_counts, _ = _sample_load_counts( - RnaSample, project, sample_type=Value('RNA'), dataset_type=F('data_type'), loadedDate=TruncDate('created_date'), - ) - - first_loaded_samples = sample_models.order_by('individual__family', 'loaded_date').distinct('individual__family').values_list('id', flat=True) - samples = sample_models.filter(Q(is_active=True) | Q(id__in=first_loaded_samples)) - samples_by_guid = {s['sampleGuid']: s for s in get_json_for_samples(samples, project_guid=project_guid)} + datasets = Dataset.objects.filter( + Q(active_individuals__family__project=project) | Q(inactive_individuals__family__project=project) + ).distinct() + datasets_by_guid = {d['datasetGuid']: d for d in get_json_for_datasets(datasets, project_guid)} + rna_sample_load_counts = RnaSample.objects.filter( + individual__family__project=project, + ).values('data_type', loadedDate=TruncDate('created_date')).order_by('loadedDate').annotate( + familyCounts=ArrayAgg('individual__family__guid'), + ) grouped_sample_counts = defaultdict(list) - for s in sample_load_counts + rna_sample_load_counts: + for s in rna_sample_load_counts: s['familyCounts'] = {f: s['familyCounts'].count(f) for f in s['familyCounts']} - grouped_sample_counts[f'{s.pop("sample_type")}__{s.pop("dataset_type")}'].append(s) + grouped_sample_counts[s.pop('data_type')].append(s) - project_json = {'projectGuid': project_guid, 'sampleCounts': grouped_sample_counts} + project_json = {'projectGuid': project_guid, 'rnaSampleCounts': grouped_sample_counts} response = { - 'samplesByGuid': samples_by_guid, + 'datasetsByGuid': datasets_by_guid, } add_project_tag_type_counts(project, response, project_json=project_json) @@ -299,13 +298,6 @@ def project_overview(request, project_guid): return create_json_response(response) -def _sample_load_counts(sample_cls, project, *args, **kwargs): - sample_models = sample_cls.objects.filter(individual__family__project=project) - return list(sample_models.values(*args, **kwargs).order_by('loadedDate').annotate( - familyCounts=ArrayAgg('individual__family__guid')) - ), sample_models - - @login_and_policies_required def project_collaborators(request, project_guid): project = get_project_and_check_permissions(project_guid, request.user) diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 31e4b8bb4b..ba77c4db46 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -327,6 +327,7 @@ def _get_sample_json_kwargs(project_guid=None, family_guid=None, individual_guid return {'guid_key': 'sampleGuid', 'nested_fields': nested_fields, **kwargs} +# TODO deprecate def get_json_for_samples(samples, **kwargs): """Returns a JSON representation of the given list of Samples. @@ -339,6 +340,14 @@ def get_json_for_samples(samples, **kwargs): return get_json_for_queryset(samples, **_get_sample_json_kwargs(**kwargs)) +def get_json_for_datasets(datasets, project_guid): + return get_json_for_queryset(datasets, additional_values={ + 'projectGuid': Value(project_guid), + 'activeIndividuals': ArrayAgg('active_individuals__guid', filter=Q(active_individuals__isnull=False)), + 'inactiveIndividuals': ArrayAgg('inactive_individuals__guid', filter=Q(inactive_individuals__isnull=False)), + }) + + def get_json_for_sample(sample, **kwargs): """Returns a JSON representation of the given Sample. diff --git a/ui/pages/Project/components/ProjectOverview.jsx b/ui/pages/Project/components/ProjectOverview.jsx index 82ef203dad..b5567b3406 100644 --- a/ui/pages/Project/components/ProjectOverview.jsx +++ b/ui/pages/Project/components/ProjectOverview.jsx @@ -29,6 +29,7 @@ import { getProjectAnalysisGroupFamilySizeHistogram, getProjectAnalysisGroupDataLoadedFamilySizeHistogram, getProjectAnalysisGroupSamplesByTypes, + getProjectAnalysisGroupRnaSamplesByTypes, getProjectAnalysisGroupMmeSubmissionDetails, getMmeSubmissionsLoading, } from '../selectors' @@ -303,15 +304,19 @@ class DatasetSection extends React.PureComponent { } -const Dataset = React.memo(({ showLoadWorkspaceData, hasAnvil, samplesByType, user }) => { - const datasetSections = samplesByType.map(([sampleTypeKey, loadedSampleCounts]) => { +const Dataset = React.memo(({ showLoadWorkspaceData, hasAnvil, samplesByType, rnaSamplesByType, user }) => { + const datasetSections = [...Object.entries(samplesByType).map(([sampleTypeKey, loadedSampleCounts]) => { const [sampleType, datasetType] = sampleTypeKey.split('__') return { key: sampleTypeKey, title: `${SAMPLE_TYPE_LOOKUP[sampleType] || sampleType}${DATASET_TITLE_LOOKUP[datasetType] || ''} Datasets`, content: , } - }).sort((a, b) => a.title.localeCompare(b.title)) + }), ...rnaSamplesByType.map(([dataType, loadedSampleCounts]) => ({ + key: dataType, + title: `RNA${DATASET_TITLE_LOOKUP[dataType]} Datasets`, + content: , + }))].sort((a, b) => a.title.localeCompare(b.title)) const noLoadedData = !datasetSections.length if (noLoadedData) { @@ -358,6 +363,7 @@ const Dataset = React.memo(({ showLoadWorkspaceData, hasAnvil, samplesByType, us Dataset.propTypes = { samplesByType: PropTypes.object.isRequired, + rnaSamplesByType: PropTypes.arrayOf(PropTypes.object).isRequired, hasAnvil: PropTypes.bool, showLoadWorkspaceData: PropTypes.bool, user: PropTypes.object.isRequired, @@ -366,6 +372,7 @@ Dataset.propTypes = { const mapDatasetStateToProps = (state, ownProps) => ({ user: getUser(state), samplesByType: getProjectAnalysisGroupSamplesByTypes(state, ownProps), + rnaSamplesByType: getProjectAnalysisGroupRnaSamplesByTypes(state, ownProps), }) const DatasetOverview = connect(mapDatasetStateToProps)(Dataset) diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index 7fb9dafb6b..70a20cbc0b 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -15,6 +15,7 @@ import { SIMPLIFIED_SEX_LOOKUP, } from 'shared/utils/constants' import { toCamelcase, toSnakecase, snakecaseToTitlecase } from 'shared/utils/stringUtils' +import { compareObjects } from 'shared/utils/sortUtils' import { getProjectsByGuid, getFamiliesGroupedByProjectGuid, getIndividualsByGuid, getGenesById, getUser, @@ -23,7 +24,7 @@ import { getVariantTagsByGuid, getUserOptionsByUsername, getNotesByFamilyType, getVariantTagNotesByFamilyVariants, getPhenotypeGeneScoresByIndividual, getActiveDatasetsByIndividual, getRnaSeqDataByIndividual, familyPassesFilters, getAnalysisGroupGuid, getCurrentAnalysisGroupFamilyGuids, - getDatasetsByIndividual, getActiveDatasetsByFamily, getMinMaxDatasetsByFamily, + getDatasetsByIndividual, getActiveDatasetsByFamily, getMinMaxDatasetsByFamily, getDatasetsGroupedByProjectGuid, } from 'redux/selectors' import { @@ -170,9 +171,32 @@ export const getProjectAnalysisGroupIndividualsByGuid = createSelector( ) export const getProjectAnalysisGroupSamplesByTypes = createSelector( + getProjectGuid, + getCurrentAnalysisGroupFamilyGuids, + getProjectAnalysisGroupIndividualsByGuid, + getDatasetsGroupedByProjectGuid, + (projectGuid, analysisGroupFamilyGuids, analysisGroupIndividuals, datasetsByProject) => Object.values( + datasetsByProject[projectGuid] || {}, + ).sort(compareObjects('loadedDate')).reduce((acc, { sampleType, datasetType, loadedDate, activeIndividuals, inactiveIndividuals }) => { + const key = `${sampleType}__${datasetType}` + if (!acc[key]) { + acc[key] = [] + } + const individualGuids = [...activeIndividuals, ...inactiveIndividuals] + const count = !analysisGroupFamilyGuids ? individualGuids.length : individualGuids.filter( + individualGuid => individualGuid in analysisGroupIndividuals, + ).length + if (count) { + acc[key].push({ loadedDate, count }) + } + return acc + }, {}), +) + +export const getProjectAnalysisGroupRnaSamplesByTypes = createSelector( getCurrentProject, getCurrentAnalysisGroupFamilyGuids, - (project, analysisGroupFamilyGuids) => Object.entries(project.sampleCounts || {}).map( + (project, analysisGroupFamilyGuids) => Object.entries(project.rnaSampleCounts || {}).map( ([key, typeCounts]) => ([key, typeCounts.map(({ familyCounts, ...data }) => ({ ...data, count: Object.entries(familyCounts).reduce((total, [familyGuid, count]) => ( diff --git a/ui/redux/rootReducer.js b/ui/redux/rootReducer.js index d28acefe1f..1b43ca36ea 100644 --- a/ui/redux/rootReducer.js +++ b/ui/redux/rootReducer.js @@ -262,7 +262,7 @@ const rootReducer = combineReducers({ familyNotesByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'familyNotesByGuid'), familyDetailsLoading: createSingleObjectReducer(REQUEST_FAMILY_DETAILS), individualsByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'individualsByGuid'), - samplesByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'samplesByGuid'), + datasetsByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'datasetsByGuid'), igvSamplesByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'igvSamplesByGuid'), analysisGroupsByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'analysisGroupsByGuid'), analysisGroupsLoading: loadingReducer(REQUEST_ANALYSIS_GROUPS, RECEIVE_ANALYSIS_GROUPS), diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index 3fe2aace45..00cbe8f10b 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -11,7 +11,7 @@ export const getFamiliesByGuid = state => state.familiesByGuid export const getFamilyNotesByGuid = state => state.familyNotesByGuid export const getFamilyDetailsLoading = state => state.familyDetailsLoading export const getIndividualsByGuid = state => state.individualsByGuid -const getSamplesByGuid = state => state.samplesByGuid +const getDatasetsByGuid = state => state.datasetsByGuid export const getIgvSamplesByGuid = state => state.igvSamplesByGuid export const getAnalysisGroupsByGuid = state => state.analysisGroupsByGuid export const getAnalysisGroupIsLoading = state => state.analysisGroupsLoading.isLoading @@ -56,6 +56,7 @@ const groupEntitiesByProjectGuid = entities => Object.entries(entities).reduce(( }, {}) export const getFamiliesGroupedByProjectGuid = createSelector(getFamiliesByGuid, groupEntitiesByProjectGuid) export const getAnalysisGroupsGroupedByProjectGuid = createSelector(getAnalysisGroupsByGuid, groupEntitiesByProjectGuid) +export const getDatasetsGroupedByProjectGuid = createSelector(getDatasetsByGuid, groupEntitiesByProjectGuid) const groupByFamilyGuid = objs => objs.reduce((acc, o) => { if (!acc[o.familyGuid]) { @@ -67,14 +68,14 @@ const groupByFamilyGuid = objs => objs.reduce((acc, o) => { export const getProjectDatasetTypes = createSelector( getProjectsByGuid, - getSamplesByGuid, - (projectsByGuid, samplesByGuid) => { + getDatasetsByGuid, + (projectsByGuid, datasetsByGuid) => { const projectDatasetTypes = Object.values(projectsByGuid).reduce( (acc, { projectGuid, datasetTypes }) => ({ ...acc, [projectGuid]: datasetTypes }), {}, ) - const sampleDatasetTypes = Object.values(samplesByGuid).reduce( - (acc, { projectGuid, datasetType, isActive }) => { - if (projectDatasetTypes[projectGuid] || !isActive) { + const sampleDatasetTypes = Object.values(datasetsByGuid).reduce( + (acc, { projectGuid, datasetType, activeIndividuals }) => { + if (projectDatasetTypes[projectGuid] || !activeIndividuals) { return acc } if (!acc[projectGuid]) { @@ -89,15 +90,23 @@ export const getProjectDatasetTypes = createSelector( ) export const getDatasetsByIndividual = createSelector( - getSamplesByGuid, - samplesByGuid => Object.values(samplesByGuid).sort( + getDatasetsByGuid, + datasetsByGuid => Object.values(datasetsByGuid).sort( (a, b) => a.loadedDate.localeCompare(b.loadedDate), - ).reduce((acc, sample) => { - const { individualGuid, isActive, sampleType, datasetType, loadedDate } = sample - if (!acc[individualGuid]) { - acc[individualGuid] = [] - } - acc[individualGuid].push({ isActive, sampleType, datasetType, loadedDate: loadedDate.split('T')[0] }) + ).reduce((acc, { activeIndividuals, inactiveIndividuals, sampleType, datasetType, loadedDate }) => { + const parsedLoadedDate = loadedDate.split('T')[0] + activeIndividuals.forEach((individualGuid) => { + if (!acc[individualGuid]) { + acc[individualGuid] = [] + } + acc[individualGuid].push({ isActive: true, sampleType, datasetType, loadedDate: parsedLoadedDate }) + }) + inactiveIndividuals.forEach((individualGuid) => { + if (!acc[individualGuid]) { + acc[individualGuid] = [] + } + acc[individualGuid].push({ isActive: false, sampleType, datasetType, loadedDate: parsedLoadedDate }) + }) return acc }, {}), ) From 6ccc339f4de06505ca08998430d2dff4cd6e6c73 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 10 Apr 2026 16:27:34 -0400 Subject: [PATCH 027/119] use datasets on family page --- seqr/views/apis/family_api.py | 14 +++++++------- seqr/views/utils/orm_to_json_utils.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 50c378ab81..0b934912cf 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -17,11 +17,11 @@ from seqr.views.utils.json_to_orm_utils import update_family_from_json, update_model_from_json, create_model_from_json from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.note_utils import create_note_handler, update_note_handler, delete_note_handler -from seqr.views.utils.orm_to_json_utils import _get_json_for_model, get_json_for_family_note, get_json_for_samples, \ +from seqr.views.utils.orm_to_json_utils import _get_json_for_model, get_json_for_family_note, get_json_for_datasets, \ get_json_for_matchmaker_submissions, get_json_for_analysis_groups, _get_json_for_families, get_json_for_queryset from seqr.views.utils.project_context_utils import add_families_context, families_discovery_tags, add_project_tag_types, \ MME_TAG_NAME -from seqr.models import Family, FamilyAnalysedBy, Individual, FamilyNote, Sample, Dataset, VariantTag, AnalysisGroup, RnaSeqTpm, \ +from seqr.models import Family, FamilyAnalysedBy, Individual, FamilyNote, Dataset, VariantTag, AnalysisGroup, RnaSeqTpm, \ PhenotypePrioritization, Project, RnaSample from seqr.views.utils.permissions_utils import check_project_permissions, get_project_and_check_pm_permissions, \ login_and_policies_required, user_is_analyst, has_case_review_permissions, external_anvil_project_can_edit, \ @@ -45,12 +45,12 @@ def family_page_data(request, family_guid): is_analyst = user_is_analyst(request.user) has_case_review_perm = has_case_review_permissions(project, request.user) - sample_models = Sample.objects.filter(individual__family=family) - samples = get_json_for_samples( - sample_models, project_guid=project.guid, family_guid=family_guid, is_analyst=is_analyst - ) + dataset_models = Dataset.objects.filter( + Q(active_individuals__family=family) | Q(inactive_individuals__family=family) + ).distinct() + datasets = get_json_for_datasets(dataset_models, project.guid) response = { - 'samplesByGuid': {s['sampleGuid']: s for s in samples} + 'datasetsByGuid': {d['datasetGuid']: d for d in datasets} } add_families_context(response, families, project.guid, request.user, is_analyst, has_case_review_perm) diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index ba77c4db46..56b1101094 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -343,8 +343,8 @@ def get_json_for_samples(samples, **kwargs): def get_json_for_datasets(datasets, project_guid): return get_json_for_queryset(datasets, additional_values={ 'projectGuid': Value(project_guid), - 'activeIndividuals': ArrayAgg('active_individuals__guid', filter=Q(active_individuals__isnull=False)), - 'inactiveIndividuals': ArrayAgg('inactive_individuals__guid', filter=Q(inactive_individuals__isnull=False)), + 'activeIndividuals': ArrayAgg('active_individuals__guid', distinct=True, filter=Q(active_individuals__isnull=False)), + 'inactiveIndividuals': ArrayAgg('inactive_individuals__guid', distinct=True, filter=Q(inactive_individuals__isnull=False)), }) From e67833eb04dbd2126873c83cafc535798500b3f2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 10 Apr 2026 16:32:37 -0400 Subject: [PATCH 028/119] remove sample loading --- seqr/urls.py | 3 +-- seqr/views/apis/project_api.py | 12 +----------- seqr/views/utils/orm_to_json_utils.py | 1 - ui/pages/Project/reducers.js | 3 --- 4 files changed, 2 insertions(+), 17 deletions(-) diff --git a/seqr/urls.py b/seqr/urls.py index ab0b4b461f..4b2e11a7d9 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -144,7 +144,7 @@ from seqr.views.apis.project_api import create_project_handler, update_project_handler, delete_project_handler, \ project_page_data, project_families, project_overview, project_mme_submisssions, project_individuals, \ project_analysis_groups, update_project_workspace, project_family_notes, project_collaborators, project_locus_lists, \ - project_samples, project_notifications, mark_read_project_notifications, subscribe_project_notifications, \ + project_notifications, mark_read_project_notifications, subscribe_project_notifications, \ update_project_rna_seq, load_rna_seq_sample_data from seqr.views.apis.project_categories_api import update_project_categories_handler from seqr.views.apis.anvil_workspace_api import anvil_workspace_page, create_project_from_workspace, \ @@ -212,7 +212,6 @@ 'project/(?P[^/]+)/details': project_page_data, 'project/(?P[^/]+)/get_families': project_families, 'project/(?P[^/]+)/get_individuals': project_individuals, - 'project/(?P[^/]+)/get_samples': project_samples, 'project/(?P[^/]+)/get_family_notes': project_family_notes, 'project/(?P[^/]+)/get_mme_submissions': project_mme_submisssions, 'project/(?P[^/]+)/get_analysis_groups': project_analysis_groups, diff --git a/seqr/views/apis/project_api.py b/seqr/views/apis/project_api.py index eca9d9d0d3..4641d331c9 100644 --- a/seqr/views/apis/project_api.py +++ b/seqr/views/apis/project_api.py @@ -21,7 +21,7 @@ from seqr.views.utils.individual_utils import delete_individuals from seqr.views.utils.json_utils import create_json_response, _to_snake_case, _to_camel_case from seqr.views.utils.json_to_orm_utils import update_project_from_json, create_model_from_json, update_model_from_json -from seqr.views.utils.orm_to_json_utils import _get_json_for_project, get_json_for_samples, get_json_for_datasets, \ +from seqr.views.utils.orm_to_json_utils import _get_json_for_project, get_json_for_datasets, \ get_json_for_project_collaborator_list, get_json_for_matchmaker_submissions, \ get_json_for_family_notes, _get_json_for_individuals, get_json_for_project_collaborator_groups, \ FAMILY_ADDITIONAL_VALUES @@ -322,16 +322,6 @@ def project_individuals(request, project_guid): }) -@login_and_policies_required -def project_samples(request, project_guid): - project = get_project_and_check_permissions(project_guid, request.user) - samples = Sample.objects.filter(individual__family__project=project) - - return create_json_response({ - 'samplesByGuid': {s['sampleGuid']: s for s in get_json_for_samples(samples, project_guid=project_guid)}, - }) - - @login_and_policies_required def project_analysis_groups(request, project_guid): project = get_project_and_check_permissions(project_guid, request.user) diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 56b1101094..b067a5a564 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -327,7 +327,6 @@ def _get_sample_json_kwargs(project_guid=None, family_guid=None, individual_guid return {'guid_key': 'sampleGuid', 'nested_fields': nested_fields, **kwargs} -# TODO deprecate def get_json_for_samples(samples, **kwargs): """Returns a JSON representation of the given list of Samples. diff --git a/ui/pages/Project/reducers.js b/ui/pages/Project/reducers.js index c2181b8dfe..9d0d355724 100644 --- a/ui/pages/Project/reducers.js +++ b/ui/pages/Project/reducers.js @@ -56,11 +56,8 @@ export const loadMmeSubmissions = () => loadCurrentProjectChildEntities('mme sub const loadFamilyNotes = () => loadCurrentProjectChildEntities('family notes', REQUEST_FAMILIES, RECEIVE_FAMILIES) -const loadSamples = () => loadCurrentProjectChildEntities('samples', REQUEST_INDIVIDUALS) - export const loadProjectExportData = () => (dispatch, getState) => Promise.all([ loadIndividuals()(dispatch, getState), - loadSamples()(dispatch, getState), loadFamilyNotes()(dispatch, getState), ]) From 43947d790bd9962d45f614dac9714df27e4c6360 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 10 Apr 2026 16:42:52 -0400 Subject: [PATCH 029/119] fix tests --- seqr/views/apis/family_api_tests.py | 23 ++++++++------ seqr/views/apis/project_api_tests.py | 47 ++++++---------------------- seqr/views/utils/test_utils.py | 5 ++- 3 files changed, 25 insertions(+), 50 deletions(-) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index e8fa545231..381a29e323 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -15,7 +15,7 @@ family_variant_tag_summary, update_family_analysis_groups, get_family_rna_seq_data, get_family_phenotype_gene_scores from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, \ FAMILY_NOTE_FIELDS, FAMILY_FIELDS, IGV_SAMPLE_FIELDS, \ - SAMPLE_FIELDS, INDIVIDUAL_FIELDS, INTERNAL_INDIVIDUAL_FIELDS, INTERNAL_FAMILY_FIELDS, CASE_REVIEW_FAMILY_FIELDS, \ + DATASET_FIELDS, INDIVIDUAL_FIELDS, INTERNAL_INDIVIDUAL_FIELDS, INTERNAL_FAMILY_FIELDS, CASE_REVIEW_FAMILY_FIELDS, \ MATCHMAKER_SUBMISSION_FIELDS, TAG_TYPE_FIELDS, CASE_REVIEW_INDIVIDUAL_FIELDS from seqr.models import FamilyAnalysedBy, AnalysisGroup, Individual @@ -49,7 +49,7 @@ def test_family_page_data(self): response_json = response.json() response_keys = { - 'familiesByGuid', 'individualsByGuid', 'familyNotesByGuid', 'samplesByGuid', 'igvSamplesByGuid', + 'familiesByGuid', 'individualsByGuid', 'familyNotesByGuid', 'datasetsByGuid', 'igvSamplesByGuid', 'mmeSubmissionsByGuid', } self.assertSetEqual(set(response_json.keys()), response_keys) @@ -74,7 +74,7 @@ def test_family_page_data(self): self.assertEqual(len(response_json['individualsByGuid']), 3) individual = response_json['individualsByGuid'][INDIVIDUAL_GUID] - individual_fields = {'sampleGuids', 'igvSampleGuids', 'mmeSubmissionGuid', 'phenotypePrioritizationTools', 'rnaSample'} + individual_fields = {'igvSampleGuids', 'mmeSubmissionGuid', 'phenotypePrioritizationTools', 'rnaSample'} individual_fields.update(INDIVIDUAL_FIELDS) self.assertSetEqual(set(individual.keys()), individual_fields) self.assertListEqual([ @@ -96,12 +96,15 @@ def test_family_page_data(self): self.assertSetEqual({PROJECT_GUID}, {i['projectGuid'] for i in response_json['individualsByGuid'].values()}) self.assertSetEqual({FAMILY_GUID}, {i['familyGuid'] for i in response_json['individualsByGuid'].values()}) - self.assertEqual(len(response_json['samplesByGuid']), 3) - self.assertSetEqual(set(next(iter(response_json['samplesByGuid'].values())).keys()), SAMPLE_FIELDS) - self.assertSetEqual({PROJECT_GUID}, {s['projectGuid'] for s in response_json['samplesByGuid'].values()}) - self.assertSetEqual({FAMILY_GUID}, {s['familyGuid'] for s in response_json['samplesByGuid'].values()}) - self.assertEqual(len(individual['sampleGuids']), 1) - self.assertTrue(set(individual['sampleGuids']).issubset(set(response_json['samplesByGuid'].keys()))) + self.assertEqual(len(response_json['datasetsByGuid']), 2) + dataset = response_json['datasetsByGuid']['S000129_na19675'] + self.assertSetEqual(set(dataset.keys()), DATASET_FIELDS) + self.assertSetEqual({PROJECT_GUID}, {s['projectGuid'] for s in response_json['datasetsByGuid'].values()}) + self.assertListEqual( + dataset['activeIndividuals'], + ['I000001_na19675', 'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000007_na20870', 'I000009_na20874'], + ) + self.assertListEqual(dataset['inactiveIndividuals'], ['I000003_na19679']) self.assertEqual(len(response_json['igvSamplesByGuid']), 1) self.assertSetEqual(set(next(iter(response_json['igvSamplesByGuid'].values())).keys()), IGV_SAMPLE_FIELDS) @@ -210,7 +213,7 @@ def test_family_page_data(self): self.assertEqual(family['projectGuid'], PROJECT_GUID) self.assertSetEqual(set(family['individualGuids']), set(response_json['individualsByGuid'].keys())) self.assertEqual(len(response_json['individualsByGuid']), 3) - self.assertEqual(len(response_json['samplesByGuid']), 7) + self.assertEqual(len(response_json['datasetsByGuid']), 3) self.assertEqual(len(response_json['igvSamplesByGuid']), 0) self.assertEqual(len(response_json['mmeSubmissionsByGuid']), 0) self.assertEqual(len(response_json['familyNotesByGuid']), 0) diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index 43b4fcf6ad..96b5538d38 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -12,7 +12,7 @@ from seqr.views.apis.project_api import create_project_handler, delete_project_handler, update_project_handler, \ project_page_data, project_families, project_overview, project_mme_submisssions, project_individuals, \ project_analysis_groups, update_project_workspace, project_family_notes, project_collaborators, project_locus_lists, \ - project_samples, project_notifications, mark_read_project_notifications, subscribe_project_notifications, \ + project_notifications, mark_read_project_notifications, subscribe_project_notifications, \ update_project_rna_seq, load_rna_seq_sample_data from seqr.views.apis.data_manager_api_tests import RNA_OUTLIER_SAMPLE_DATA, RNA_OUTLIER_MUSCLE_SAMPLE_GUID, RNA_TPM_SAMPLE_DATA, \ RNA_TPM_MUSCLE_SAMPLE_GUID, RNA_SPLICE_SAMPLE_DATA, RNA_SPLICE_SAMPLE_GUID, PLACEHOLDER_GUID, \ @@ -20,7 +20,7 @@ from seqr.views.utils.terra_api_utils import TerraAPIException, TerraRefreshTokenFailedException from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, \ PROJECT_FIELDS, LOCUS_LIST_FIELDS, PA_LOCUS_LIST_FIELDS, NO_INTERNAL_CASE_REVIEW_INDIVIDUAL_FIELDS, \ - SAMPLE_FIELDS, SUMMARY_FAMILY_FIELDS, INTERNAL_INDIVIDUAL_FIELDS, INDIVIDUAL_FIELDS, TAG_TYPE_FIELDS, \ + DATASET_FIELDS, SUMMARY_FAMILY_FIELDS, INTERNAL_INDIVIDUAL_FIELDS, INDIVIDUAL_FIELDS, TAG_TYPE_FIELDS, \ FAMILY_NOTE_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, ANALYSIS_GROUP_FIELDS, \ EXT_WORKSPACE_NAMESPACE, TEST_EMPTY_PROJECT_WORKSPACE, DYNAMIC_ANALYSIS_GROUP_FIELDS @@ -341,12 +341,12 @@ def test_project_overview(self): response_json = response.json() response_keys = { - 'projectsByGuid', 'samplesByGuid', 'familyTagTypeCounts', + 'projectsByGuid', 'datasetsByGuid', 'familyTagTypeCounts', } self.assertSetEqual(set(response_json.keys()), response_keys) project_fields = { - 'variantTagTypes', 'variantFunctionalTagTypes', 'sampleCounts', + 'variantTagTypes', 'variantFunctionalTagTypes', 'rnaSampleCounts', 'projectGuid', 'mmeDeletedSubmissionCount', 'mmeSubmissionCount', } project_response = response_json['projectsByGuid'][PROJECT_GUID] @@ -374,25 +374,16 @@ def test_project_overview(self): 'order': 99, 'numTags': 1, }) - self.assertDictEqual(project_response['sampleCounts'], { - 'WES__SNV_INDEL': [{ - 'familyCounts': { - 'F000001_1': 3, 'F000002_2': 3, 'F000003_3': 1, 'F000004_4': 1, 'F000005_5': 1, 'F000006_6': 1, - 'F000007_7': 1, 'F000008_8': 1, 'F000010_10': 1, - }, - 'loadedDate': '2017-02-05', - }], - 'WES__SV': [{'familyCounts': {'F000002_2': 3}, 'loadedDate': '2018-02-05'}], - 'WES__MITO': [{'familyCounts': {'F000002_2': 1}, 'loadedDate': '2022-02-05'}], - 'RNA__S': [{'familyCounts': {'F000001_1': 2}, 'loadedDate': '2017-02-05'}], - 'RNA__T': [{'familyCounts': {'F000001_1': 2}, 'loadedDate': '2017-02-05'}], - 'RNA__E': [{'familyCounts': {'F000001_1': 1}, 'loadedDate': '2017-02-05'}], + self.assertDictEqual(project_response['rnaSampleCounts'], { + 'S': [{'familyCounts': {'F000001_1': 2}, 'loadedDate': '2017-02-05'}], + 'T': [{'familyCounts': {'F000001_1': 2}, 'loadedDate': '2017-02-05'}], + 'E': [{'familyCounts': {'F000001_1': 1}, 'loadedDate': '2017-02-05'}], }) self.assertEqual(project_response['mmeSubmissionCount'], 1) self.assertEqual(project_response['mmeDeletedSubmissionCount'], 0) - self.assertEqual(len(response_json['samplesByGuid']), 16) - self.assertSetEqual(set(next(iter(response_json['samplesByGuid'].values())).keys()), SAMPLE_FIELDS) + self.assertEqual(len(response_json['datasetsByGuid']), 5) + self.assertSetEqual(set(next(iter(response_json['datasetsByGuid'].values())).keys()), DATASET_FIELDS) self.assertDictEqual(response_json['familyTagTypeCounts'], { 'F000001_1': {'Review': 1, 'Tier 1 - Novel gene and phenotype': 1, 'MME Submission': 1}, 'F000002_2': {'AIP': 1, 'Excluded': 1, 'Known gene for phenotype': 1}, @@ -543,24 +534,6 @@ def test_project_individuals(self): NO_INTERNAL_CASE_REVIEW_INDIVIDUAL_FIELDS, ) - def test_project_samples(self): - url = reverse(project_samples, args=[PROJECT_GUID]) - self.check_collaborator_login(url) - - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - - response_json = response.json() - response_keys = {'samplesByGuid'} - self.assertSetEqual(set(response_json.keys()), response_keys) - - self.assertEqual(len(response_json['samplesByGuid']), 17) - self.assertSetEqual(set(next(iter(response_json['samplesByGuid'].values())).keys()), SAMPLE_FIELDS) - - # Test empty project - empty_url = reverse(project_samples, args=[EMPTY_PROJECT_GUID]) - self._check_empty_project(empty_url, response_keys) - def test_project_analysis_groups(self): url = reverse(project_analysis_groups, args=[PROJECT_GUID]) self.check_collaborator_login(url) diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index 1be9e8a56a..64a6b308e6 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -703,9 +703,8 @@ def _get_list_param(call, param): INTERNAL_INDIVIDUAL_FIELDS = deepcopy(NO_INTERNAL_CASE_REVIEW_INDIVIDUAL_FIELDS) INTERNAL_INDIVIDUAL_FIELDS.update(CORE_INTERNAL_INDIVIDUAL_FIELDS) -SAMPLE_FIELDS = { - 'projectGuid', 'familyGuid', 'individualGuid', 'sampleGuid', 'createdDate', 'sampleType', 'sampleId', 'isActive', - 'loadedDate', 'datasetType', +DATASET_FIELDS = { + 'projectGuid', 'datasetGuid', 'sampleType', 'loadedDate', 'datasetType', 'activeIndividuals', 'inactiveIndividuals', } IGV_SAMPLE_FIELDS = { From 15a6cae67cb38737624998db53c742fbbb0ee7a1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 10 Apr 2026 16:44:19 -0400 Subject: [PATCH 030/119] actually remove sample table --- seqr/migrations/0087_dataset.py | 3 ++ seqr/models.py | 53 --------------------------------- 2 files changed, 3 insertions(+), 53 deletions(-) diff --git a/seqr/migrations/0087_dataset.py b/seqr/migrations/0087_dataset.py index 7bc5cb367f..1ae541adf6 100644 --- a/seqr/migrations/0087_dataset.py +++ b/seqr/migrations/0087_dataset.py @@ -111,4 +111,7 @@ class Migration(migrations.Migration): ), migrations.RunPython(set_individual_id), migrations.RunPython(populate_datasets), + migrations.DeleteModel( + name='Sample', + ), ] diff --git a/seqr/models.py b/seqr/models.py index 47551870a0..64cf2bb572 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -694,59 +694,6 @@ class Meta: audit_fields = {'case_review_status'} -class Sample(ModelWithGUID): - """This model represents a single data type (eg. Variant Calls, or SV Calls) that's generated from a single - biological sample (eg. WES, WGS). - - It stores metadata on both the dataset (fields: dataset_type, loaded_date, etc.) and the underlying sample - (fields: sample_type, sample_id etc.) - """ - - SAMPLE_TYPE_WES = 'WES' - SAMPLE_TYPE_WGS = 'WGS' - SAMPLE_TYPE_CHOICES = ( - (SAMPLE_TYPE_WES, 'Exome'), - (SAMPLE_TYPE_WGS, 'Whole Genome'), - ) - SAMPLE_TYPE_LOOKUP = dict(SAMPLE_TYPE_CHOICES) - - DATASET_TYPE_VARIANT_CALLS = 'SNV_INDEL' - DATASET_TYPE_SV_CALLS = 'SV' - DATASET_TYPE_MITO_CALLS = 'MITO' - DATASET_TYPE_CHOICES = ( - (DATASET_TYPE_VARIANT_CALLS, 'Variant Calls'), - (DATASET_TYPE_SV_CALLS, 'SV Calls'), - (DATASET_TYPE_MITO_CALLS, 'Mitochondria calls'), - ) - DATASET_TYPE_LOOKUP = dict(DATASET_TYPE_CHOICES) - - individual = models.ForeignKey('Individual', on_delete=models.PROTECT) - - sample_type = models.CharField(max_length=10, choices=SAMPLE_TYPE_CHOICES) - dataset_type = models.CharField(max_length=13, choices=DATASET_TYPE_CHOICES) - - # The sample's id in the underlying dataset (eg. the VCF Id for variant callsets). - sample_id = models.TextField(db_index=True) - - elasticsearch_index = models.TextField(db_index=True, null=True) - data_source = models.TextField(null=True) - - # sample status - is_active = models.BooleanField(default=False) - loaded_date = models.DateTimeField() - - def __unicode__(self): - return self.sample_id.strip() - - GUID_PREFIX = 'S' - GUID_PRECISION = 10 - - class Meta: - json_fields = [ - 'guid', 'created_date', 'sample_type', 'dataset_type', 'sample_id', 'is_active', 'loaded_date', - ] - - class Dataset(ModelWithGUID): SAMPLE_TYPE_WES = 'WES' SAMPLE_TYPE_WGS = 'WGS' From a683e65328a829ab79f9b10b32c6fe8859d42d32 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 16 Apr 2026 16:40:10 -0400 Subject: [PATCH 031/119] update js search data --- ui/pages/Project/fixtures.js | 15 +++------ ui/pages/Search/fixtures.js | 45 ++++++-------------------- ui/shared/components/panel/fixtures.js | 2 +- 3 files changed, 16 insertions(+), 46 deletions(-) diff --git a/ui/pages/Project/fixtures.js b/ui/pages/Project/fixtures.js index 29a4152ea8..a24c69994b 100644 --- a/ui/pages/Project/fixtures.js +++ b/ui/pages/Project/fixtures.js @@ -121,7 +121,7 @@ export const STATE1 = { sex: 'M', }, }, - samplesByGuid: {}, + datasetsByGuid: {}, mmeSubmissionsByGuid: {}, project: { createdDate: '2016-05-16T05:37:08.634Z', @@ -310,19 +310,14 @@ export const STATE_WITH_2_FAMILIES = { sex: 'M', }, }, - samplesByGuid: { + datasetsByGuid: { S2310656_wal_mc16200_mc16203: { - createdDate: "2018-03-30T11:50:40.079Z", - datasetFilePath: "gs://seqr-datasets/GRCh37/cmg_sankaran_wes/CMG_MYOSEQ.vcf.gz", - datasetName: null, datasetType: "SNV_INDEL", - familyGuid: 'F011652_2', - individualGuid: "I021476_na19678_2", + activeIndividuals: ["I021476_na19678_2"], + inactiveIndividuals: [], loadedDate: "2018-03-13T13:25:21.551Z", projectGuid: "R0237_1000_genomes_demo", - sampleGuid: "S2310656_wal_mc16200_mc16203", - sampleId: "WAL_MC16200_MC16203", - isActive: true, + datasetGuid: "S2310656_wal_mc16200_mc16203", sampleType: "WES", }, }, diff --git a/ui/pages/Search/fixtures.js b/ui/pages/Search/fixtures.js index 49bbe7a4f2..c277cedfd1 100644 --- a/ui/pages/Search/fixtures.js +++ b/ui/pages/Search/fixtures.js @@ -158,57 +158,32 @@ export const STATE = { paternalId: '', }, }, - samplesByGuid: { + datasetsByGuid: { S2310658_wal_mc16200_mc16203: { - createdDate: "2018-03-30T11:50:40.079Z", - elasticsearchIndex: "CMG_MYOSEQ", datasetType: "SNV_INDEL", - individualGuid: "I021476_na19678", + activeIndividuals: ["I021476_na19678"], + inactiveIndividuals: ["I021476_na19678"], loadedDate: "2018-03-13T13:25:21.551Z", projectGuid: PROJECT_GUID, - familyGuid: FAMILY_GUID, - sampleGuid: "S2310656_wal_mc16200_mc16203", - sampleId: "WAL_MC16200_MC16203", - isActive: true, + datasetGuid: "S2310656_wal_mc16200_mc16203", sampleType: "WES", }, S2310657_wal_mc16200_mc16203: { - createdDate: "2018-03-30T11:50:40.079Z", - elasticsearchIndex: "CMG_MYOSEQ", datasetType: "SV", - individualGuid: "I021476_na19678", - loadedDate: "2018-03-13T13:25:21.551Z", - projectGuid: PROJECT_GUID, - sampleGuid: "S2310656_wal_mc16200_mc16203", - sampleId: "WAL_MC16200_MC16203", - isActive: true, - sampleType: "WES", - }, - S2310656_wal_mc16200_mc16203: { - createdDate: "2018-03-30T11:50:40.079Z", - elasticsearchIndex: "CMG_MYOSEQ", - datasetType: "SNV_INDEL", - individualGuid: "I021476_na19678", + activeIndividuals: ["I021476_na19678"], + inactiveIndividuals: [], loadedDate: "2018-03-13T13:25:21.551Z", projectGuid: PROJECT_GUID, - familyGuid: FAMILY_GUID, - sampleGuid: "S2310656_wal_mc16200_mc16203", - sampleId: "WAL_MC16200_MC16203", - isActive: false, + datasetGuid: "S2310656_wal_mc16200_mc16203", sampleType: "WES", }, S2310659_wal_mc16200_mc16203: { - createdDate: "2018-03-30T11:50:40.079Z", - elasticsearchIndex: null, - datasetName: null, datasetType: "SNV_INDEL", - individualGuid: "I021476_na19678", + activeIndividuals: ["I021476_na19678"], + inactiveIndividuals: [], loadedDate: "2018-03-13T13:25:21.551Z", projectGuid: PROJECT_GUID, - familyGuid: FAMILY_GUID, - sampleGuid: "S2310656_wal_mc16200_mc16203", - sampleId: "WAL_MC16200_MC16203", - isActive: true, + datasetGuid: "S2310656_wal_mc16200_mc16203", sampleType: "RNA", }, }, diff --git a/ui/shared/components/panel/fixtures.js b/ui/shared/components/panel/fixtures.js index 7765dfd8d5..96232431c1 100644 --- a/ui/shared/components/panel/fixtures.js +++ b/ui/shared/components/panel/fixtures.js @@ -329,7 +329,7 @@ export const STATE1 = { }, }, analysisGroupsByGuid: {}, - samplesByGuid: {}, + datasetsByGuid: {}, igvSamplesByGuid: { S2310656_wal_mc16200_mc16203: { projectGuid: 'R0237_1000_genomes_demo', From d331d1400b2ac22b98fb5980db1c0e15b54580d1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 16 Apr 2026 16:45:37 -0400 Subject: [PATCH 032/119] fix test setup --- clickhouse_search/search_tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 155eef4f92..a3be2aa078 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -406,7 +406,6 @@ def test_both_sample_types_search(self): project_families=MULTI_PROJECT_PROJECT_FAMILIES, check_login=self.check_collaborator_login, ) - self._add_sample_type_samples('WGS', guid='S000129_na19675') dataset = Dataset.objects.get(guid='S000129_na19675') dataset.pk = None dataset.sample_type = 'WGS' From f9ee1d939fff44f76a57bb8acf5b700976c6eac3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 16 Apr 2026 16:52:54 -0400 Subject: [PATCH 033/119] remove sample guid from response --- seqr/views/utils/individual_utils.py | 2 +- seqr/views/utils/orm_to_json_utils.py | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index 816b58ed4a..69018b8f55 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -252,7 +252,7 @@ def _get_updated_pedigree_json(updated_individuals, updated_families, updated_no individual['individualGuid']: individual for individual in _get_json_for_individuals(Individual.objects.filter(id__in=[ i.id for i in updated_individuals - ]), user, add_sample_guids_field=True) + ]), user) } families_by_guid = { family['familyGuid']: family for family in diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index dd3454a579..2e9affdacb 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -264,7 +264,7 @@ def get_json_for_family_note(note): INDIVIDUAL_DISPLAY_NAME_EXPR = Coalesce(NullIf('display_name', Value('')), 'individual_id', output_field=CharField()) -def _get_json_for_individuals(individuals, user=None, project_guid=None, add_sample_guids_field=False, +def _get_json_for_individuals(individuals, user=None, project_guid=None, add_hpo_details=False, is_analyst=None, has_case_review_perm=False): additional_model_fields = _get_case_review_fields(individuals.model, has_case_review_perm) nested_fields = [ @@ -283,11 +283,6 @@ def _get_json_for_individuals(individuals, user=None, project_guid=None, add_sam 'paternalId': F('father__individual_id'), 'displayName': INDIVIDUAL_DISPLAY_NAME_EXPR, } - if add_sample_guids_field: - additional_values.update({ - f'{field}Guids': ArrayAgg(f'{field.lower()}__guid', filter=Q(**{f'{field.lower()}__isnull': False})) - for field in ['sample', 'igvSample'] - }) parsed_individuals = get_json_for_queryset( individuals, user=user, is_analyst=is_analyst, additional_values=additional_values, From 823ada275bd225a5414ca80ae81090928187d4e9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 16 Apr 2026 17:06:02 -0400 Subject: [PATCH 034/119] fix test --- seqr/views/apis/anvil_workspace_api_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/apis/anvil_workspace_api_tests.py b/seqr/views/apis/anvil_workspace_api_tests.py index 00fd6639a4..2a2c5427f1 100644 --- a/seqr/views/apis/anvil_workspace_api_tests.py +++ b/seqr/views/apis/anvil_workspace_api_tests.py @@ -806,7 +806,7 @@ def _assert_valid_operation(self, project, test_add_data=True): self.mock_gzip_temp_open.assert_called_with(f'{TEMP_PATH}/db_id_to_gene_id.csv.gz', 'wt') gene_file = self.mock_gzip_temp_open.return_value.__enter__.return_value.write.call_args.args[0].split('\n') - self.assertEqual(len(gene_file), 52) + self.assertEqual(len(gene_file), 59) self.assertListEqual(gene_file[:3], ['db_id,gene_id', '1,ENSG00000223972', '2,ENSG00000227232']) gs_path = f'gs://seqr-loading-temp/v3.1/{genome_version}/SNV_INDEL/pedigrees/WES/' From 030522a8e67567f1ec37f1231f0a5df422418ff0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 16 Apr 2026 17:21:43 -0400 Subject: [PATCH 035/119] fix distinct --- clickhouse_search/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 4021de06a0..3f5ba71435 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -618,7 +618,7 @@ def _get_sample_metadata(individuals, affected_family_only, annotate_affected_ma annotations['samples'] = ArrayAgg(JSONObject( affected='affected', sex='sex', sample_id='individual_id', sample_type='active_datasets__sample_type', family_guid=F('family__guid'), individual_guid=F('guid'), - )) + ), distinct=True) return individuals.aggregate(**annotations) From 0668305126a65a282623548d568605fa78869820 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 16 Apr 2026 17:43:32 -0400 Subject: [PATCH 036/119] undo --- seqr/views/apis/anvil_workspace_api_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/apis/anvil_workspace_api_tests.py b/seqr/views/apis/anvil_workspace_api_tests.py index 2a2c5427f1..00fd6639a4 100644 --- a/seqr/views/apis/anvil_workspace_api_tests.py +++ b/seqr/views/apis/anvil_workspace_api_tests.py @@ -806,7 +806,7 @@ def _assert_valid_operation(self, project, test_add_data=True): self.mock_gzip_temp_open.assert_called_with(f'{TEMP_PATH}/db_id_to_gene_id.csv.gz', 'wt') gene_file = self.mock_gzip_temp_open.return_value.__enter__.return_value.write.call_args.args[0].split('\n') - self.assertEqual(len(gene_file), 59) + self.assertEqual(len(gene_file), 52) self.assertListEqual(gene_file[:3], ['db_id,gene_id', '1,ENSG00000223972', '2,ENSG00000227232']) gs_path = f'gs://seqr-loading-temp/v3.1/{genome_version}/SNV_INDEL/pedigrees/WES/' From 9cd739af9a9b3f837a104a60310499e3e708a654 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 16 Apr 2026 17:52:47 -0400 Subject: [PATCH 037/119] more random otpions --- clickhouse_search/search_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index a3be2aa078..cc8a58c125 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -139,7 +139,7 @@ def assert_cached_results(self, expected_results, cache_key): self.mock_redis.expire.reset_mock() def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=None, request_body=None, check_login=None, query_params=None, search_hash=None, **search_kwargs): - search_hash = search_hash or random.randint(1000, 10000) # nosec + search_hash = search_hash or random.randint(1000, 9000000) # nosec self.mock_results_guid.return_value = f'VRS{search_hash:07d}' url = reverse(query_variants_handler, args=[search_hash]) From 757f63d6765fd30759256fad022917d2cd60a1b6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 24 Apr 2026 13:39:43 -0400 Subject: [PATCH 038/119] pr feedback --- ui/pages/Project/components/ProjectOverview.jsx | 2 +- ui/pages/Project/selectors.js | 2 +- ui/pages/Search/fixtures.js | 15 +++++++-------- ui/redux/selectors.js | 8 +++++--- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/ui/pages/Project/components/ProjectOverview.jsx b/ui/pages/Project/components/ProjectOverview.jsx index b5567b3406..be47568655 100644 --- a/ui/pages/Project/components/ProjectOverview.jsx +++ b/ui/pages/Project/components/ProjectOverview.jsx @@ -363,7 +363,7 @@ const Dataset = React.memo(({ showLoadWorkspaceData, hasAnvil, samplesByType, rn Dataset.propTypes = { samplesByType: PropTypes.object.isRequired, - rnaSamplesByType: PropTypes.arrayOf(PropTypes.object).isRequired, + rnaSamplesByType: PropTypes.arrayOf(PropTypes.any).isRequired, hasAnvil: PropTypes.bool, showLoadWorkspaceData: PropTypes.bool, user: PropTypes.object.isRequired, diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index 70a20cbc0b..5dd1c3252b 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -182,7 +182,7 @@ export const getProjectAnalysisGroupSamplesByTypes = createSelector( if (!acc[key]) { acc[key] = [] } - const individualGuids = [...activeIndividuals, ...inactiveIndividuals] + const individualGuids = [...(activeIndividuals || []), ...(inactiveIndividuals || [])] const count = !analysisGroupFamilyGuids ? individualGuids.length : individualGuids.filter( individualGuid => individualGuid in analysisGroupIndividuals, ).length diff --git a/ui/pages/Search/fixtures.js b/ui/pages/Search/fixtures.js index c277cedfd1..013130e72e 100644 --- a/ui/pages/Search/fixtures.js +++ b/ui/pages/Search/fixtures.js @@ -162,29 +162,28 @@ export const STATE = { S2310658_wal_mc16200_mc16203: { datasetType: "SNV_INDEL", activeIndividuals: ["I021476_na19678"], - inactiveIndividuals: ["I021476_na19678"], + inactiveIndividuals: [], loadedDate: "2018-03-13T13:25:21.551Z", projectGuid: PROJECT_GUID, - datasetGuid: "S2310656_wal_mc16200_mc16203", + datasetGuid: "S2310658_wal_mc16200_mc16203", sampleType: "WES", }, S2310657_wal_mc16200_mc16203: { datasetType: "SV", activeIndividuals: ["I021476_na19678"], - inactiveIndividuals: [], + inactiveIndividuals: null, loadedDate: "2018-03-13T13:25:21.551Z", projectGuid: PROJECT_GUID, - datasetGuid: "S2310656_wal_mc16200_mc16203", + datasetGuid: "S2310657_wal_mc16200_mc16203", sampleType: "WES", }, - S2310659_wal_mc16200_mc16203: { + S2310656_wal_mc16200_mc16203: { datasetType: "SNV_INDEL", - activeIndividuals: ["I021476_na19678"], - inactiveIndividuals: [], + inactiveIndividuals: ["I021476_na19678"], loadedDate: "2018-03-13T13:25:21.551Z", projectGuid: PROJECT_GUID, datasetGuid: "S2310656_wal_mc16200_mc16203", - sampleType: "RNA", + sampleType: "WES", }, }, analysisGroupsByGuid: { diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index 78e2891385..51fbfb4b04 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -76,7 +76,7 @@ export const getProjectDatasetTypes = createSelector( ) const sampleDatasetTypes = Object.values(datasetsByGuid).reduce( (acc, { projectGuid, datasetType, activeIndividuals }) => { - if (projectDatasetTypes[projectGuid] || !activeIndividuals) { + if (projectDatasetTypes[projectGuid] || !(activeIndividuals || []).length) { return acc } if (!acc[projectGuid]) { @@ -96,13 +96,15 @@ export const getDatasetsByIndividual = createSelector( (a, b) => a.loadedDate.localeCompare(b.loadedDate), ).reduce((acc, { activeIndividuals, inactiveIndividuals, sampleType, datasetType, loadedDate }) => { const parsedLoadedDate = loadedDate.split('T')[0] - activeIndividuals.forEach((individualGuid) => { + const activeIndivs = activeIndividuals || [] + activeIndivs.forEach((individualGuid) => { if (!acc[individualGuid]) { acc[individualGuid] = [] } acc[individualGuid].push({ isActive: true, sampleType, datasetType, loadedDate: parsedLoadedDate }) }) - inactiveIndividuals.forEach((individualGuid) => { + const inactiveIndivs = inactiveIndividuals || [] + inactiveIndivs.forEach((individualGuid) => { if (!acc[individualGuid]) { acc[individualGuid] = [] } From 52a9af62023a4e4fbe7d6ba8d57453155c4549e9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 24 Apr 2026 13:53:19 -0400 Subject: [PATCH 039/119] fix merge --- clickhouse_search/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 55e389488c..1d80c43dcb 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -1033,7 +1033,7 @@ def delete_clickhouse_project(project, dataset_type, sample_type=None): table_base = f'{GENOME_VERSION_LOOKUP[project.genome_version]}/{dataset_type}' partition_id = f"'{project.guid}'" partition_ids = [partition_id] - if project.genome_version == GENOME_VERSION_GRCh38 and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: + if project.genome_version == GENOME_VERSION_GRCh38 and dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS: partition_ids = [ f'({partition_id}, {pid})' for pid in EntriesSnvIndel.objects.filter(project_guid=project.guid).values_list('partition_id', flat=True).distinct() From 36f3dae83e6b8fcf27eda2a7d84cc06084f14f9d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 27 Apr 2026 12:22:16 -0400 Subject: [PATCH 040/119] move unneccessarily nested file --- seqr/management/commands/check_for_new_samples_from_pipeline.py | 2 +- .../commands/transfer_families_to_different_project.py | 2 +- seqr/utils/{search => }/add_data_utils.py | 0 seqr/views/apis/anvil_workspace_api.py | 2 +- seqr/views/apis/data_manager_api.py | 2 +- seqr/views/utils/dataset_utils.py | 2 +- seqr/views/utils/individual_utils.py | 2 +- 7 files changed, 6 insertions(+), 6 deletions(-) rename seqr/utils/{search => }/add_data_utils.py (100%) diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py index 3ce9618d97..abe2fd3ecf 100644 --- a/seqr/management/commands/check_for_new_samples_from_pipeline.py +++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py @@ -13,7 +13,7 @@ from seqr.models import Family, Sample, Project, Individual, SavedVariant from seqr.utils.communication_utils import safe_post_to_slack, send_project_email from seqr.utils.file_utils import file_iter, list_files, is_google_bucket_file_path -from seqr.utils.search.add_data_utils import notify_search_data_loaded, update_airtable_loading_tracking_status +from seqr.utils.add_data_utils import notify_search_data_loaded, update_airtable_loading_tracking_status from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS from seqr.views.utils.export_utils import write_multiple_files from seqr.views.utils.permissions_utils import is_internal_anvil_project, project_has_anvil diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py index 356a0a615c..93ac4ba1ae 100644 --- a/seqr/management/commands/transfer_families_to_different_project.py +++ b/seqr/management/commands/transfer_families_to_different_project.py @@ -1,7 +1,7 @@ from django.core.management.base import BaseCommand from seqr.models import Project, Family, VariantTag, VariantTagType -from seqr.utils.search.add_data_utils import trigger_delete_families_search +from seqr.utils.add_data_utils import trigger_delete_families_search import logging logger = logging.getLogger(__name__) diff --git a/seqr/utils/search/add_data_utils.py b/seqr/utils/add_data_utils.py similarity index 100% rename from seqr/utils/search/add_data_utils.py rename to seqr/utils/add_data_utils.py diff --git a/seqr/views/apis/anvil_workspace_api.py b/seqr/views/apis/anvil_workspace_api.py index 5106343a75..24c2152510 100644 --- a/seqr/views/apis/anvil_workspace_api.py +++ b/seqr/views/apis/anvil_workspace_api.py @@ -21,7 +21,7 @@ from seqr.views.utils.pedigree_info_utils import parse_basic_pedigree_table, JsonConstants from seqr.views.utils.individual_utils import add_or_update_individuals_and_families from seqr.utils.file_utils import list_files -from seqr.utils.search.add_data_utils import get_missing_family_samples, get_loaded_individual_ids, trigger_data_loading +from seqr.utils.add_data_utils import get_missing_family_samples, get_loaded_individual_ids, trigger_data_loading from seqr.utils.vcf_utils import validate_vcf_and_get_samples, get_vcf_list from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 6f0fc97ae9..422a1b2aec 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -11,7 +11,7 @@ from clickhouse_search.search import delete_clickhouse_project from seqr.utils.communication_utils import send_project_notification -from seqr.utils.search.add_data_utils import trigger_data_loading, get_missing_family_samples, get_loaded_individual_ids, trigger_delete_families_search +from seqr.utils.add_data_utils import trigger_data_loading, get_missing_family_samples, get_loaded_individual_ids, trigger_delete_families_search from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException from seqr.utils.vcf_utils import validate_vcf_and_get_samples, get_vcf_list diff --git a/seqr/views/utils/dataset_utils.py b/seqr/views/utils/dataset_utils.py index f0a5571cf2..5a3714f326 100644 --- a/seqr/views/utils/dataset_utils.py +++ b/seqr/views/utils/dataset_utils.py @@ -11,7 +11,7 @@ from seqr.utils.file_utils import file_iter, is_google_bucket_file_path, run_gsutil_with_wait from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException -from seqr.utils.search.add_data_utils import basic_notify_search_data_loaded +from seqr.utils.add_data_utils import basic_notify_search_data_loaded from seqr.utils.xpos_utils import format_chrom from seqr.views.utils.file_utils import parse_file, get_temp_file_path, persist_temp_file from seqr.views.utils.json_utils import _to_snake_case, _to_camel_case diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index b9e46d1a21..d12cf24b06 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -7,7 +7,7 @@ from matchmaker.models import MatchmakerSubmission, MatchmakerResult from seqr.models import Sample, IgvSample, RnaSample, Individual, Family, FamilyNote from seqr.utils.middleware import ErrorsWarningsException -from seqr.utils.search.add_data_utils import trigger_rebuild_gt_stats +from seqr.utils.add_data_utils import trigger_rebuild_gt_stats from seqr.views.utils.json_to_orm_utils import update_individual_from_json, update_individual_parents, create_model_from_json, \ update_family_from_json from seqr.views.utils.orm_to_json_utils import _get_json_for_individuals, _get_json_for_families, get_json_for_family_notes From c00347d75183d850739f62a838e879b5388f1f7b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 27 Apr 2026 12:24:47 -0400 Subject: [PATCH 041/119] move liftover helper --- clickhouse_search/search.py | 35 ++++++++++++++++++++++++++++++++--- seqr/utils/search/utils.py | 32 -------------------------------- 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index d3368c7457..61b3bf1feb 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -7,6 +7,7 @@ from django.db.models.functions import JSONObject from django.db.utils import OperationalError import json +from pyliftover.liftover import LiftOver from clickhouse_search.backend.fields import NamedTupleField from clickhouse_search.backend.functions import Array, ArrayFilter, ArrayIntersect, ArraySort, GroupArrayArray, If, Tuple, \ @@ -16,7 +17,7 @@ from clickhouse_search.models.reference_data_models import BaseClinvar, BaseHgmd from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, EntriesSnvIndel, \ ENTRY_CLASS_MAP, VARIANTS_CLASS_MAP, VARIANT_DETAILS_CLASS_MAP -from reference_data.models import GeneInfo, GeneConstraint, Omim, GENOME_VERSION_LOOKUP, GENOME_VERSION_GRCh38 +from reference_data.models import GeneInfo, GeneConstraint, Omim, GENOME_VERSION_LOOKUP, GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 from seqr.models import Sample, PhenotypePrioritization, Family, Individual from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.logging_utils import SeqrLogger @@ -926,9 +927,8 @@ def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, aff lifted_genome_version = next(gv for gv in ENTRY_CLASS_MAP.keys() if gv != genome_version) lifted_entry_cls = ENTRY_CLASS_MAP[lifted_genome_version].get(data_type) if lifted_entry_cls: - from seqr.utils.search.utils import run_liftover chrom, pos, base_id = variant_id.split('-', 2) - liftover_results = run_liftover(lifted_genome_version, chrom, int(pos)) + liftover_results = _run_liftover(lifted_genome_version, chrom, int(pos)) if liftover_results: lifted_id = f'{liftover_results[0]}-{liftover_results[1]}-{base_id}' entry_qs = lifted_entry_cls.objects.filter_locus(raw_variant_items=lifted_id) @@ -1050,3 +1050,32 @@ def delete_clickhouse_project(project, dataset_type, sample_type=None): PROJECT_GT_STATS_VIEW_CLASS_MAP[project.genome_version][dataset_type].refresh() ENTRY_CLASS_MAP[project.genome_version][dataset_type].gt_stats.rel.related_model.reload() return f'Deleted all {dataset_type} search data for project {project.name}' + + +LIFTOVERS = { + GENOME_VERSION_GRCh38: None, + GENOME_VERSION_GRCh37: None, +} +PYLIFTOVER_BUILD_LOOKUP = { + GENOME_VERSION_GRCh38: ('hg19', 'hg38'), + GENOME_VERSION_GRCh37: ('hg38', 'hg19'), +} +def _get_liftover(genome_version): + if not LIFTOVERS[genome_version]: + try: + LIFTOVERS[genome_version] = LiftOver(*PYLIFTOVER_BUILD_LOOKUP[genome_version]) + except Exception as e: + logger.error('ERROR: Unable to set up liftover. {}'.format(e), user=None) + return LIFTOVERS[genome_version] + + +def _run_liftover(genome_version, chrom, pos): + liftover = _get_liftover(genome_version) + if not liftover: + return None + lifted_coord = liftover.convert_coordinate( + 'chr{}'.format(chrom.lstrip('chr')), int(pos) + ) + if lifted_coord and lifted_coord[0]: + return (lifted_coord[0][0].lstrip('chr'), lifted_coord[0][1]) + return None diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 8df369fd04..89ff20292c 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -2,11 +2,8 @@ from copy import deepcopy from datetime import timedelta -from pyliftover.liftover import LiftOver - from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ get_sorted_search_results, clickhouse_variant_lookup, InvalidSearchException -from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 from seqr.models import Sample, VariantSearchResults from seqr.utils.logging_utils import SeqrLogger from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json @@ -141,32 +138,3 @@ def get_variant_query_gene_counts(search_model, user): for family_guid in var['familyGuids']: gene_aggs[gene_id]['families'][family_guid] += 1 return gene_aggs - - -LIFTOVERS = { - GENOME_VERSION_GRCh38: None, - GENOME_VERSION_GRCh37: None, -} -PYLIFTOVER_BUILD_LOOKUP = { - GENOME_VERSION_GRCh38: ('hg19', 'hg38'), - GENOME_VERSION_GRCh37: ('hg38', 'hg19'), -} -def _get_liftover(genome_version): - if not LIFTOVERS[genome_version]: - try: - LIFTOVERS[genome_version] = LiftOver(*PYLIFTOVER_BUILD_LOOKUP[genome_version]) - except Exception as e: - logger.error('ERROR: Unable to set up liftover. {}'.format(e), user=None) - return LIFTOVERS[genome_version] - - -def run_liftover(genome_version, chrom, pos): - liftover = _get_liftover(genome_version) - if not liftover: - return None - lifted_coord = liftover.convert_coordinate( - 'chr{}'.format(chrom.lstrip('chr')), int(pos) - ) - if lifted_coord and lifted_coord[0]: - return (lifted_coord[0][0].lstrip('chr'), lifted_coord[0][1]) - return None From 03669cb5b64b7e72f997f6cadd0ffad0f3d5e10c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 27 Apr 2026 12:35:33 -0400 Subject: [PATCH 042/119] move get single variant --- seqr/utils/search/utils.py | 9 +-------- seqr/views/apis/variant_search_api.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 89ff20292c..7f820db33f 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -17,13 +17,6 @@ MAX_EXPORT_VARIANTS = 1000 -def get_single_variant(families, variant_id, user): - variants = get_clickhouse_variants(families, user, raw_variant_items=variant_id, variant_ids=[variant_id]) - if not variants: - raise InvalidSearchException('Variant {} not found'.format(variant_id)) - return format_clickhouse_results(variants)[0] - - def variant_lookup(user, variant_id, genome_version, sample_type=None, affected_only=False, hom_only=False): cache_fields = ['variant_lookup_results', variant_id, genome_version] if affected_only: @@ -75,7 +68,7 @@ def query_variants(search_model, sort, page, num_results, user): sort = PATHOGENICTY_HGMD_SORT_KEY all_results = _query_variants(search_model, user, sort=sort or XPOS_SORT_KEY) - results_page = format_clickhouse_results(all_results[(page-1)*num_results:page*num_results]) + results_page = all_results[(page-1)*num_results:page*num_results] return results_page, len(all_results) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 6f7013eb8f..523baba194 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -12,11 +12,12 @@ from math import ceil import re +from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ + get_sorted_search_results, clickhouse_variant_lookup, InvalidSearchException from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_LOOKUP from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Sample -from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, \ +from seqr.utils.search.utils import query_variants, get_variant_query_gene_counts, \ variant_lookup, export_variants -from seqr.utils.search.utils import InvalidSearchException from seqr.views.utils.export_utils import export_table from seqr.utils.gene_utils import get_genes_for_variant_display from seqr.utils.logging_utils import SeqrLogger @@ -142,9 +143,11 @@ def query_single_variant_handler(request, variant_id): family = families.first() check_project_permissions(family.project, request.user) - variant = get_single_variant(families, variant_id, request.user) + variants = get_clickhouse_variants(families, request.user, raw_variant_items=variant_id, variant_ids=[variant_id]) + if not variants: + raise InvalidSearchException('Variant {} not found'.format(variant_id)) - response = _process_variants([variant], families, request, add_all_context=True, add_locus_list_detail=True) + response = _process_variants(variants, families, request, add_all_context=True, add_locus_list_detail=True) return create_json_response(response) @@ -153,6 +156,7 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo if not variants: return {'searchedVariantIds': [], 'variantsById': {}} + variants = format_clickhouse_results(variants) flat_variants = _flatten_variants(variants) variants_by_id = {v['variantId']: v for v in flat_variants} saved_variants = _get_saved_variant_models(flat_variants) From 39473e0e56ab441fa3c669ac3be3a38bea59a91f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 27 Apr 2026 12:52:33 -0400 Subject: [PATCH 043/119] clean up variant lookup --- clickhouse_search/search_tests.py | 2 +- seqr/utils/search/utils.py | 17 ----------------- seqr/views/apis/variant_search_api.py | 23 +++++++++++++++++++++-- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 6307fce27e..e4f7cb994f 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -981,7 +981,7 @@ def test_invalid_search(self): response = self.client.get(export_url) self.assertEqual(response.status_code, 403) - @mock.patch('seqr.utils.search.utils.LiftOver') + @mock.patch('clickhouse_search.search.LiftOver') def test_variant_lookup(self, mock_liftover): mock_convert_coordinate = mock_liftover.return_value.convert_coordinate mock_convert_coordinate.side_effect = lambda chrom, pos: [(chrom, pos + 10000)] diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 7f820db33f..f43d7303ea 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -17,23 +17,6 @@ MAX_EXPORT_VARIANTS = 1000 -def variant_lookup(user, variant_id, genome_version, sample_type=None, affected_only=False, hom_only=False): - cache_fields = ['variant_lookup_results', variant_id, genome_version] - if affected_only: - cache_fields.append('affected') - if hom_only: - cache_fields.append('hom') - cache_key = '__'.join(cache_fields) - variants = safe_redis_get_json(cache_key) - if variants: - return variants - - variants = clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, affected_only, hom_only) - - safe_redis_set_json(cache_key, variants, expire=timedelta(weeks=2)) - return variants - - def _get_search_cache_key(search_model, sort=None): return 'search_results__{}__{}'.format(search_model.guid, sort or XPOS_SORT_KEY) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 523baba194..6ad1b057a1 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -2,6 +2,7 @@ import jmespath from collections import defaultdict from copy import deepcopy +from datetime import timedelta from django.utils import timezone from django.contrib.postgres.aggregates import ArrayAgg from django.core.exceptions import MultipleObjectsReturned, PermissionDenied @@ -17,10 +18,11 @@ from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_LOOKUP from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Sample from seqr.utils.search.utils import query_variants, get_variant_query_gene_counts, \ - variant_lookup, export_variants + export_variants from seqr.views.utils.export_utils import export_table from seqr.utils.gene_utils import get_genes_for_variant_display from seqr.utils.logging_utils import SeqrLogger +from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json from seqr.utils.xpos_utils import parse_variant_id from seqr.views.utils.json_utils import create_json_response, _to_snake_case from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ @@ -532,7 +534,7 @@ def variant_lookup_handler(request): variant_id = request.GET.get('variantId') genome_version = request.GET.get('genomeVersion') or GENOME_VERSION_GRCh38 bool_kwargs = {_to_snake_case(field): bool(request.GET.get(field)) for field in ['affectedOnly', 'homOnly']} - variants = variant_lookup(request.user, variant_id, genome_version, sample_type=request.GET.get('sampleType'), **bool_kwargs) + variants = _cached_variant_lookup(request.user, variant_id, genome_version, sample_type=request.GET.get('sampleType'), **bool_kwargs) family_guids = set() for variant in variants: @@ -560,6 +562,23 @@ def variant_lookup_handler(request): return create_json_response(response) +def _cached_variant_lookup(user, variant_id, genome_version, sample_type, affected_only, hom_only): + cache_fields = ['variant_lookup_results', variant_id, genome_version] + if affected_only: + cache_fields.append('affected') + if hom_only: + cache_fields.append('hom') + cache_key = '__'.join(cache_fields) + variants = safe_redis_get_json(cache_key) + if variants: + return variants + + variants = clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, affected_only, hom_only) + + safe_redis_set_json(cache_key, variants, expire=timedelta(weeks=2)) + return variants + + def _update_lookup_variant(variant, response, individual_guid_map, user): no_access_families = set(variant['familyGenotypes']) - set(variant['familyGuids']) individual_summary_map = { From 63a20de606b061e1854624dc04f83504752f68a3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 27 Apr 2026 14:53:41 -0400 Subject: [PATCH 044/119] clean up cache helper --- seqr/views/apis/variant_search_api.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 6ad1b057a1..43267f1555 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -74,6 +74,15 @@ def query_variants_handler(request, search_hash): return create_json_response(response) +def _get_variants_with_cache(get_cache_key, get_variants, *args, **kwargs): + cache_key = get_cache_key(*args, **kwargs) + variants = safe_redis_get_json(cache_key) + if not variants: + variants = get_variants(*args, **kwargs) + safe_redis_set_json(cache_key, variants, expire=timedelta(weeks=2)) + return variants + + def _all_project_family_search_genome(search_context): return (search_context or {}).get('allGenomeProjectFamilies') @@ -534,7 +543,10 @@ def variant_lookup_handler(request): variant_id = request.GET.get('variantId') genome_version = request.GET.get('genomeVersion') or GENOME_VERSION_GRCh38 bool_kwargs = {_to_snake_case(field): bool(request.GET.get(field)) for field in ['affectedOnly', 'homOnly']} - variants = _cached_variant_lookup(request.user, variant_id, genome_version, sample_type=request.GET.get('sampleType'), **bool_kwargs) + variants = _get_variants_with_cache( + _get_lookup_cache_key, clickhouse_variant_lookup, + request.user, variant_id, sample_type=request.GET.get('sampleType'), genome_version=genome_version, **bool_kwargs, + ) family_guids = set() for variant in variants: @@ -562,21 +574,13 @@ def variant_lookup_handler(request): return create_json_response(response) -def _cached_variant_lookup(user, variant_id, genome_version, sample_type, affected_only, hom_only): +def _get_lookup_cache_key(user, variant_id, sample_type, genome_version, affected_only, hom_only): cache_fields = ['variant_lookup_results', variant_id, genome_version] if affected_only: cache_fields.append('affected') if hom_only: cache_fields.append('hom') - cache_key = '__'.join(cache_fields) - variants = safe_redis_get_json(cache_key) - if variants: - return variants - - variants = clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, affected_only, hom_only) - - safe_redis_set_json(cache_key, variants, expire=timedelta(weeks=2)) - return variants + return '__'.join(cache_fields) def _update_lookup_variant(variant, response, individual_guid_map, user): From 79a8b8d2001fd44f98650f92438347eca6b20b2e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 28 Apr 2026 16:30:40 -0400 Subject: [PATCH 045/119] clean up search logic --- seqr/utils/search/utils.py | 52 --------------------------- seqr/views/apis/variant_search_api.py | 36 ++++++++++++++++--- 2 files changed, 32 insertions(+), 56 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index f43d7303ea..8eb2d2607c 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -17,10 +17,6 @@ MAX_EXPORT_VARIANTS = 1000 -def _get_search_cache_key(search_model, sort=None): - return 'search_results__{}__{}'.format(search_model.guid, sort or XPOS_SORT_KEY) - - def export_variants(search_model, user): search_results = _query_variants(search_model, user) total_variants = len(search_results) @@ -29,54 +25,6 @@ def export_variants(search_model, user): return format_clickhouse_export_results(search_results) -def _get_previous_search_results(search_model, sort): - previous_search_results = None - if sort: - cache_key = _get_search_cache_key(search_model, sort=sort) - previous_search_results = safe_redis_get_json(cache_key) or [] - if not previous_search_results: - wildcard_cache_key = _get_search_cache_key(search_model, sort='*') - previous_search_results = safe_redis_get_wildcard_json(wildcard_cache_key) - if previous_search_results and sort: - previous_search_results = get_sorted_search_results( - previous_search_results, sort, families=search_model.families.all(), - ) - cache_key = _get_search_cache_key(search_model, sort=sort) - safe_redis_set_json(cache_key, previous_search_results, expire=timedelta(weeks=2)) - return previous_search_results - - -def query_variants(search_model, sort, page, num_results, user): - if sort == PATHOGENICTY_SORT_KEY and user_is_analyst(user): - sort = PATHOGENICTY_HGMD_SORT_KEY - all_results = _query_variants(search_model, user, sort=sort or XPOS_SORT_KEY) - - results_page = all_results[(page-1)*num_results:page*num_results] - - return results_page, len(all_results) - - -def _query_variants(search_model, user, sort=None): - previous_search_results = _get_previous_search_results(search_model, sort) or {} - if previous_search_results: - return previous_search_results - - search = deepcopy(search_model.variant_search.search) - families = search_model.families.all() - - exclude_search = {} - exclude_previous_hash = (search.get('exclude') or {}).pop('previousSearchHash', None) - if exclude_previous_hash: - exclude_search = _get_clickhouse_exclude_keys(exclude_previous_hash, user) - - results = get_clickhouse_variants(families, user, sort=sort, **search, **exclude_search) - - cache_key = _get_search_cache_key(search_model, sort=sort) - safe_redis_set_json(cache_key, results, expire=timedelta(weeks=2)) - - return results - - def _get_clickhouse_exclude_keys(search_hash, user): previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash) results = _query_variants(previous_search_model, user) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index fc7ada41d3..3e7e2d932d 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -13,11 +13,12 @@ from math import ceil import re +from clickhouse_search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ get_sorted_search_results, clickhouse_variant_lookup, InvalidSearchException from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_LOOKUP from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Sample -from seqr.utils.search.utils import query_variants, get_variant_query_gene_counts, \ +from seqr.utils.search.utils import get_variant_query_gene_counts, \ export_variants from seqr.views.utils.export_utils import export_table from seqr.utils.gene_utils import get_genes_for_variant_display @@ -30,7 +31,7 @@ from seqr.views.utils.orm_to_json_utils import get_json_for_saved_variants_with_tags, get_json_for_saved_search,\ get_json_for_saved_searches, add_individual_hpo_details, FAMILY_ADDITIONAL_VALUES from seqr.views.utils.permissions_utils import check_project_permissions, get_project_guids_user_can_view, \ - login_and_policies_required, check_user_created_object_permissions, check_projects_view_permission + login_and_policies_required, check_user_created_object_permissions, check_projects_view_permission, user_is_analyst from seqr.views.utils.project_context_utils import get_projects_child_entities from seqr.views.utils.variant_utils import get_variants_response from seqr.views.utils.vlm_utils import vlm_lookup @@ -55,6 +56,9 @@ def query_variants_handler(request, search_hash): """ page = int(request.GET.get('page') or 1) per_page = int(request.GET.get('per_page') or 100) + sort = request.GET.get('sort') or XPOS_SORT_KEY + if sort == PATHOGENICTY_SORT_KEY and user_is_analyst(request.user): + sort = PATHOGENICTY_HGMD_SORT_KEY search_context = json.loads(request.body or '{}') try: @@ -64,12 +68,15 @@ def query_variants_handler(request, search_hash): _check_results_permission(results_model, request.user) - variants, total_results = query_variants(results_model, sort=request.GET.get('sort'), page=page, num_results=per_page, user=request.user) + all_variants = _get_variants_with_cache( + _get_search_cache_key, _query_variants, results_model, request.user, sort, + ) + variants = all_variants[(page-1)*per_page:page*per_page] response = _process_variants(variants or [], results_model.families.all(), request, genome_version=results_model.variant_search.search.get('no_access_project_genome_version')) response['search'] = _get_search_context(results_model) - response['search']['totalResults'] = total_results + response['search']['totalResults'] = len(all_variants) return create_json_response(response) @@ -83,6 +90,27 @@ def _get_variants_with_cache(get_cache_key, get_variants, *args, **kwargs): return variants +def _get_search_cache_key(results_model, user, sort): + return 'search_results__{}__{}'.format(results_model.guid, sort or XPOS_SORT_KEY) + + +def _query_variants(results_model, user, sort): + families = results_model.families.all() + wildcard_cache_key = _get_search_cache_key(results_model, user, sort='*') + unsorted_variants = safe_redis_get_wildcard_json(wildcard_cache_key) + if unsorted_variants: + return get_sorted_search_results(unsorted_variants, sort, families) + + search = deepcopy(results_model.variant_search.search) + + exclude_search = {} + exclude_previous_hash = (search.get('exclude') or {}).pop('previousSearchHash', None) + if exclude_previous_hash: + exclude_search = _get_clickhouse_exclude_keys(exclude_previous_hash, user) + + return get_clickhouse_variants(families, user, sort=sort, **search, **exclude_search) + + def _all_project_family_search_genome(search_context): return (search_context or {}).get('allGenomeProjectFamilies') From 3eef62218446e311f7cddaaa6ea7b49367c9b0fb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 28 Apr 2026 16:36:24 -0400 Subject: [PATCH 046/119] move gene counts --- seqr/utils/search/utils.py | 24 -------------------- seqr/views/apis/variant_search_api.py | 32 ++++++++++++++++++++------- 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 8eb2d2607c..2b0d350bfd 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -14,16 +14,6 @@ logger = SeqrLogger(__name__) -MAX_EXPORT_VARIANTS = 1000 - - -def export_variants(search_model, user): - search_results = _query_variants(search_model, user) - total_variants = len(search_results) - if total_variants > MAX_EXPORT_VARIANTS: - raise InvalidSearchException(f'Unable to export more than {MAX_EXPORT_VARIANTS} variants ({total_variants} requested)') - return format_clickhouse_export_results(search_results) - def _get_clickhouse_exclude_keys(search_hash, user): previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash) @@ -48,17 +38,3 @@ def variant_dataset_type(variant): return f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' return Sample.DATASET_TYPE_MITO_CALLS if 'mitomapPathogenic' in variant else Sample.DATASET_TYPE_VARIANT_CALLS - -def get_variant_query_gene_counts(search_model, user): - results = _query_variants(search_model, user) - flat_variants = [ - v for variants in results for v in (variants if isinstance(variants, list) else [variants]) - ] - gene_aggs = defaultdict(lambda: {'total': 0, 'families': defaultdict(int)}) - for var in flat_variants: - gene_ids = var['transcripts'].keys() if 'transcripts' in var else {t['geneId'] for t in var['sortedTranscriptConsequences']} - for gene_id in gene_ids: - gene_aggs[gene_id]['total'] += 1 - for family_guid in var['familyGuids']: - gene_aggs[gene_id]['families'][family_guid] += 1 - return gene_aggs diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 3e7e2d932d..d59ab76783 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -18,8 +18,6 @@ get_sorted_search_results, clickhouse_variant_lookup, InvalidSearchException from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_LOOKUP from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Sample -from seqr.utils.search.utils import get_variant_query_gene_counts, \ - export_variants from seqr.views.utils.export_utils import export_table from seqr.utils.gene_utils import get_genes_for_variant_display from seqr.utils.logging_utils import SeqrLogger @@ -69,7 +67,7 @@ def query_variants_handler(request, search_hash): _check_results_permission(results_model, request.user) all_variants = _get_variants_with_cache( - _get_search_cache_key, _query_variants, results_model, request.user, sort, + _get_search_cache_key, _query_variants, results_model, request.user, sort=sort, ) variants = all_variants[(page-1)*per_page:page*per_page] @@ -90,11 +88,11 @@ def _get_variants_with_cache(get_cache_key, get_variants, *args, **kwargs): return variants -def _get_search_cache_key(results_model, user, sort): - return 'search_results__{}__{}'.format(results_model.guid, sort or XPOS_SORT_KEY) +def _get_search_cache_key(results_model, user, sort=XPOS_SORT_KEY): + return 'search_results__{}__{}'.format(results_model.guid, sort) -def _query_variants(results_model, user, sort): +def _query_variants(results_model, user, sort=XPOS_SORT_KEY): families = results_model.families.all() wildcard_cache_key = _get_search_cache_key(results_model, user, sort='*') unsorted_variants = safe_redis_get_wildcard_json(wildcard_cache_key) @@ -276,6 +274,7 @@ def _process_variants(variants, families, request, add_all_context=False, add_lo {'header': 'ab'}, ] +MAX_EXPORT_VARIANTS = 1000 MAX_FAMILIES_PER_ROW = 1000 @@ -284,7 +283,19 @@ def get_variant_gene_breakdown(request, search_hash): results_model = VariantSearchResults.objects.get(search_hash=search_hash) projects = _check_results_permission(results_model, request.user) - gene_counts = get_variant_query_gene_counts(results_model, user=request.user) + results = _get_variants_with_cache(_get_search_cache_key, _query_variants, results_model, request.user) + flat_variants = [ + v for variants in results for v in (variants if isinstance(variants, list) else [variants]) + ] + gene_counts = defaultdict(lambda: {'total': 0, 'families': defaultdict(int)}) + for var in flat_variants: + gene_ids = var['transcripts'].keys() if 'transcripts' in var else {t['geneId'] for t in + var['sortedTranscriptConsequences']} + for gene_id in gene_ids: + gene_counts[gene_id]['total'] += 1 + for family_guid in var['familyGuids']: + gene_counts[gene_id]['families'][family_guid] += 1 + return create_json_response({ 'searchGeneBreakdown': {search_hash: gene_counts}, 'genesById': get_genes_for_variant_display(list(gene_counts.keys()), projects.first().genome_version), @@ -301,7 +312,12 @@ def export_variants_handler(request, search_hash): families = results_model.families.all() family_ids_by_guid = {family.guid: family.family_id for family in families} - variants = export_variants(results_model, request.user) + variants = _get_variants_with_cache(_get_search_cache_key, _query_variants, results_model, request.user) + total_variants = len(variants) + if total_variants > MAX_EXPORT_VARIANTS: + raise InvalidSearchException(f'Unable to export more than {MAX_EXPORT_VARIANTS} variants ({total_variants} requested)') + + variants = format_clickhouse_export_results(variants) saved_variants = _get_saved_variant_models(variants) json_saved_variants = get_json_for_saved_variants_with_tags(saved_variants) From 4981a55d24b26d993c860f767c5007189b948de5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 28 Apr 2026 16:57:31 -0400 Subject: [PATCH 047/119] clean up remainign utils --- clickhouse_search/search_tests.py | 9 +++++- seqr/utils/middleware.py | 2 +- seqr/utils/search/__init__.py | 0 seqr/utils/search/utils.py | 40 --------------------------- seqr/views/apis/variant_search_api.py | 33 ++++++++++++++-------- seqr/views/utils/variant_utils.py | 9 +++++- 6 files changed, 38 insertions(+), 55 deletions(-) delete mode 100644 seqr/utils/search/__init__.py delete mode 100644 seqr/utils/search/utils.py diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 64b9e45905..3fd9f64ebc 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -678,7 +678,10 @@ def test_exclude_previous_search_results(self): request_body = {'projectFamilies': DEFAULT_PROJECT_FAMILIES, 'previousSearchHash': 'abc1234'} exclude = {'previousSearch': True} - response_search = {'exclude': {**exclude, 'previousSearchHash': 'abc1234'}} + response_search = { + 'exclude_keys': {'SNV_INDEL': [1, 2]}, + 'exclude_key_pairs': {'SNV_INDEL': [[2, 3]],'SV_WES': [[18, 19]]}, + } self._assert_expected_search( [[MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3], inheritance_mode='recessive', **COMP_HET_ALL_PASS_FILTERS, @@ -692,6 +695,10 @@ def test_exclude_previous_search_results(self): self.set_cache(cache_key, [ [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], [VARIANT3, VARIANT4], GCNV_VARIANT3, MITO_VARIANT3, ]) + response_search = { + 'exclude_keys': {'MITO': [8], 'SV_WES': [18]}, + 'exclude_key_pairs': {'SNV_INDEL': [[3, 4]], 'SNV_INDEL,SV_WES': [[2, 19]]}, + } self._assert_expected_search( [VARIANT2, [GCNV_VARIANT3, GCNV_VARIANT4]], exclude=exclude, **COMP_HET_ALL_PASS_FILTERS, request_body=request_body, response_search=response_search, inheritance_mode='recessive', cached_variant_fields=[ diff --git a/seqr/utils/middleware.py b/seqr/utils/middleware.py index 4f694435c9..c0dfdc5fcd 100644 --- a/seqr/utils/middleware.py +++ b/seqr/utils/middleware.py @@ -11,7 +11,7 @@ import json import traceback -from seqr.utils.search.utils import InvalidSearchException +from clickhouse_search.search import InvalidSearchException from seqr.utils.logging_utils import SeqrLogger from seqr.views.utils.json_utils import create_json_response from seqr.views.utils.terra_api_utils import TerraAPIException diff --git a/seqr/utils/search/__init__.py b/seqr/utils/search/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py deleted file mode 100644 index 2b0d350bfd..0000000000 --- a/seqr/utils/search/utils.py +++ /dev/null @@ -1,40 +0,0 @@ -from collections import defaultdict -from copy import deepcopy -from datetime import timedelta - -from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ - get_sorted_search_results, clickhouse_variant_lookup, InvalidSearchException -from seqr.models import Sample, VariantSearchResults -from seqr.utils.logging_utils import SeqrLogger -from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json -from clickhouse_search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY -from seqr.utils.xpos_utils import parse_variant_id -from seqr.views.utils.permissions_utils import user_is_analyst - -logger = SeqrLogger(__name__) - - - -def _get_clickhouse_exclude_keys(search_hash, user): - previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash) - results = _query_variants(previous_search_model, user) - exclude_keys = defaultdict(list) - exclude_key_pairs = defaultdict(list) - for variant in results: - if isinstance(variant, list): - dt1= variant_dataset_type(variant[0]) - dt2 = variant_dataset_type(variant[1]) - dataset_type = dt1 if dt1 == dt2 else ','.join(sorted([dt1, dt2])) - exclude_key_pairs[dataset_type].append(sorted([variant[0]['key'], variant[1]['key']])) - else: - dataset_type = variant_dataset_type(variant) - exclude_keys[dataset_type].append(variant['key']) - return {'exclude_keys': dict(exclude_keys), 'exclude_key_pairs': dict(exclude_key_pairs)} - - -def variant_dataset_type(variant): - if not parse_variant_id(variant['variantId']): - sample_type = Sample.SAMPLE_TYPE_WGS if 'endChrom' in variant else Sample.SAMPLE_TYPE_WES - return f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' - return Sample.DATASET_TYPE_MITO_CALLS if 'mitomapPathogenic' in variant else Sample.DATASET_TYPE_VARIANT_CALLS - diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index d59ab76783..7e94b63f6e 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -31,7 +31,7 @@ from seqr.views.utils.permissions_utils import check_project_permissions, get_project_guids_user_can_view, \ login_and_policies_required, check_user_created_object_permissions, check_projects_view_permission, user_is_analyst from seqr.views.utils.project_context_utils import get_projects_child_entities -from seqr.views.utils.variant_utils import get_variants_response +from seqr.views.utils.variant_utils import get_variants_response, variant_dataset_type from seqr.views.utils.vlm_utils import vlm_lookup logger = SeqrLogger(__name__) @@ -99,14 +99,7 @@ def _query_variants(results_model, user, sort=XPOS_SORT_KEY): if unsorted_variants: return get_sorted_search_results(unsorted_variants, sort, families) - search = deepcopy(results_model.variant_search.search) - - exclude_search = {} - exclude_previous_hash = (search.get('exclude') or {}).pop('previousSearchHash', None) - if exclude_previous_hash: - exclude_search = _get_clickhouse_exclude_keys(exclude_previous_hash, user) - - return get_clickhouse_variants(families, user, sort=sort, **search, **exclude_search) + return get_clickhouse_variants(families, user, sort=sort, **results_model.variant_search.search) def _all_project_family_search_genome(search_context): @@ -153,7 +146,7 @@ def _get_or_create_results_model(search_hash, search_context, user): search_dict = search_context.get('search', {}) if search_context.get('previousSearchHash') and (search_dict.get('exclude') or {}).get('previousSearch'): - search_dict['exclude']['previousSearchHash'] = search_context['previousSearchHash'] + search_dict.update(_get_exclude_keys(search_context['previousSearchHash'], user)) if include_no_access_projects: search_dict['no_access_project_genome_version'] = all_project_genome_version search_model = VariantSearch.objects.filter(search=search_dict).filter( @@ -172,6 +165,23 @@ def _get_or_create_results_model(search_hash, search_context, user): return results_model +def _get_exclude_keys(search_hash, user): + previous_results_model = VariantSearchResults.objects.get(search_hash=search_hash) + results = _get_variants_with_cache(_get_search_cache_key, _query_variants, previous_results_model, user) + exclude_keys = defaultdict(list) + exclude_key_pairs = defaultdict(list) + for variant in results: + if isinstance(variant, list): + dt1= variant_dataset_type(variant[0]) + dt2 = variant_dataset_type(variant[1]) + dataset_type = dt1 if dt1 == dt2 else ','.join(sorted([dt1, dt2])) + exclude_key_pairs[dataset_type].append(sorted([variant[0]['key'], variant[1]['key']])) + else: + dataset_type = variant_dataset_type(variant) + exclude_keys[dataset_type].append(variant['key']) + return {'exclude_keys': dict(exclude_keys), 'exclude_key_pairs': dict(exclude_key_pairs)} + + @login_and_policies_required def query_single_variant_handler(request, variant_id): """Search variants. @@ -289,8 +299,7 @@ def get_variant_gene_breakdown(request, search_hash): ] gene_counts = defaultdict(lambda: {'total': 0, 'families': defaultdict(int)}) for var in flat_variants: - gene_ids = var['transcripts'].keys() if 'transcripts' in var else {t['geneId'] for t in - var['sortedTranscriptConsequences']} + gene_ids = var['transcripts'].keys() if 'transcripts' in var else {t['geneId'] for t in var['sortedTranscriptConsequences']} for gene_id in gene_ids: gene_counts[gene_id]['total'] += 1 for family_guid in var['familyGuids']: diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py index eab4d543b2..4446836be6 100644 --- a/seqr/views/utils/variant_utils.py +++ b/seqr/views/utils/variant_utils.py @@ -13,8 +13,8 @@ from reference_data.models import TranscriptInfo, Omim, GENOME_VERSION_GRCh38 from seqr.models import SavedVariant, VariantSearchResults, Family, LocusList, LocusListInterval, LocusListGene, \ RnaSeqTpm, PhenotypePrioritization, Project, Sample, RnaSample, VariantTag, VariantTagType -from seqr.utils.search.utils import variant_dataset_type from seqr.utils.gene_utils import get_genes_for_variants +from seqr.utils.xpos_utils import parse_variant_id from seqr.views.utils.json_to_orm_utils import create_model_from_json from seqr.views.utils.orm_to_json_utils import get_json_for_discovery_tags, get_json_for_locus_lists, \ get_json_for_queryset, get_json_for_rna_seq_outliers, get_json_for_saved_variants_with_tags, \ @@ -63,6 +63,13 @@ def parse_saved_variant_json(variant_json, family_id): } +def variant_dataset_type(variant): + if not parse_variant_id(variant['variantId']): + sample_type = Sample.SAMPLE_TYPE_WGS if 'endChrom' in variant else Sample.SAMPLE_TYPE_WES + return f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}' + return Sample.DATASET_TYPE_MITO_CALLS if 'mitomapPathogenic' in variant else Sample.DATASET_TYPE_VARIANT_CALLS + + def _transcript_sort(gene_id, saved_variant_json, main_transcript_id): gene_transcripts = saved_variant_json['transcripts'][gene_id] is_main_gene = bool(main_transcript_id) and any(t.get('transcriptId') == main_transcript_id for t in gene_transcripts) From e6b8505eb3de93d8333b31f04b4103fc23310e16 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 28 Apr 2026 17:38:25 -0400 Subject: [PATCH 048/119] fix mock paths --- clickhouse_search/search_tests.py | 2 +- .../tests/check_for_new_samples_from_pipeline_tests.py | 6 +++--- seqr/views/apis/anvil_workspace_api_tests.py | 6 +++--- seqr/views/apis/data_manager_api_tests.py | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 3fd9f64ebc..5e94f6f2b7 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -865,7 +865,7 @@ def test_variant_id_search(self): [],locus={'rawVariantItems': VARIANT_IDS[1]}, ) - @mock.patch('seqr.utils.search.utils.MAX_EXPORT_VARIANTS', 2) + @mock.patch('seqr.views.apis.variant_search_ap.MAX_EXPORT_VARIANTS', 2) @mock.patch('clickhouse_search.search.MAX_GENES_FOR_FILTER', 2) @mock.patch('clickhouse_search.search.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) @mock.patch('clickhouse_search.search.MAX_VARIANTS', 3) diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py index 55e19a2553..86bf7e08af 100644 --- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py +++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py @@ -405,13 +405,13 @@ def set_up(self): patcher = mock.patch('seqr.utils.communication_utils.BASE_URL', SEQR_URL) patcher.start() self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.utils.search.add_data_utils.BASE_URL', SEQR_URL) + patcher = mock.patch('seqr.utils.add_data_utils.BASE_URL', SEQR_URL) patcher.start() self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.utils.search.add_data_utils.SEQR_SLACK_ANVIL_DATA_LOADING_CHANNEL', 'anvil-data-loading') + patcher = mock.patch('seqr.utils.add_data_utils.SEQR_SLACK_ANVIL_DATA_LOADING_CHANNEL', 'anvil-data-loading') patcher.start() self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.utils.search.add_data_utils.SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL', 'seqr-data-loading') + patcher = mock.patch('seqr.utils.add_data_utils.SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL', 'seqr-data-loading') patcher.start() self.addCleanup(patcher.stop) patcher = mock.patch('seqr.views.utils.airtable_utils.BASE_URL', 'https://test-seqr.org/') diff --git a/seqr/views/apis/anvil_workspace_api_tests.py b/seqr/views/apis/anvil_workspace_api_tests.py index 00fd6639a4..70266db8a4 100644 --- a/seqr/views/apis/anvil_workspace_api_tests.py +++ b/seqr/views/apis/anvil_workspace_api_tests.py @@ -521,7 +521,7 @@ def _test_get_workspace_files(self, url, response_key, expected_files, mock_subp ]) -@mock.patch('seqr.utils.search.add_data_utils.LOADING_DATASETS_DIR', 'gs://seqr-loading-temp/v3.1') +@mock.patch('seqr.utils.add_data_utils.LOADING_DATASETS_DIR', 'gs://seqr-loading-temp/v3.1') @mock.patch('reference_data.models.GeneInfo.CURRENT_VERSION', '27') class LoadAnvilDataAPITest(AnvilAuthenticationTestCase, AirtableTest): fixtures = ['users', 'social_auth', 'reference_data', '1kg_project'] @@ -546,7 +546,7 @@ def setUp(self): patcher = mock.patch('seqr.views.utils.airtable_utils.logger') self.mock_airtable_logger = patcher.start() self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.utils.search.add_data_utils.logger') + patcher = mock.patch('seqr.utils.add_data_utils.logger') self.mock_add_data_utils_logger = patcher.start() self.addCleanup(patcher.stop) patcher = mock.patch('seqr.views.apis.anvil_workspace_api.load_uploaded_file') @@ -574,7 +574,7 @@ def setUp(self): self.mock_datetime = patcher.start() self.mock_datetime.now.side_effect = lambda: datetime(2021, 3, 1, 0, 0, 0) self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.utils.search.add_data_utils.safe_post_to_slack') + patcher = mock.patch('seqr.utils.add_data_utils.safe_post_to_slack') self.mock_slack = patcher.start() self.addCleanup(patcher.stop) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 3b688ab47e..4c5295c56d 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -533,7 +533,7 @@ def test_update_rna_splice_outlier(self, *args, **kwargs): @mock.patch('seqr.views.utils.airtable_utils.BASE_URL', 'https://seqr.broadinstitute.org/') @mock.patch('seqr.utils.communication_utils.BASE_URL', 'https://test-seqr.org/') - @mock.patch('seqr.utils.search.add_data_utils.SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL', 'seqr-data-loading') + @mock.patch('seqr.utils.add_data_utils.SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL', 'seqr-data-loading') @mock.patch('seqr.views.utils.file_utils.tempfile.gettempdir', lambda: 'tmp/') @mock.patch('seqr.views.utils.dataset_utils.os.path.isfile', lambda *args: True) @mock.patch('seqr.utils.communication_utils.send_html_email') @@ -1329,7 +1329,7 @@ def setUp(self): self.mock_subprocess = patcher.start() self.mock_subprocess.side_effect = [self.mock_file_iter] self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.utils.search.add_data_utils.LOADING_DATASETS_DIR', self.TRIGGER_CALLSET_DIR) + patcher = mock.patch('seqr.utils.add_data_utils.LOADING_DATASETS_DIR', self.TRIGGER_CALLSET_DIR) patcher.start() self.addCleanup(patcher.stop) super().setUp() @@ -1449,10 +1449,10 @@ def setUp(self): self.mock_file_iter.wait.return_value = 0 self.mock_subprocess.side_effect = [self.mock_does_file_exist, self.mock_file_iter] self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.utils.search.add_data_utils.safe_post_to_slack') + patcher = mock.patch('seqr.utils.add_data_utils.safe_post_to_slack') self.mock_slack = patcher.start() self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.utils.search.add_data_utils.LOADING_DATASETS_DIR', 'gs://seqr-loading-temp/v3.1') + patcher = mock.patch('seqr.utils.add_data_utils.LOADING_DATASETS_DIR', 'gs://seqr-loading-temp/v3.1') patcher.start() self.addCleanup(patcher.stop) super().setUp() From 2d8d8161608be605d7e4ebd2ed5295c99b7e21f4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 29 Apr 2026 10:44:41 -0400 Subject: [PATCH 049/119] do not muatet search dict --- clickhouse_search/search.py | 1 - clickhouse_search/search_tests.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 9ef1440a7f..a1093606f7 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -151,7 +151,6 @@ def _parse_locus_search(locus, genome_version, search): search.update({ 'genes': genes, 'intervals': intervals, 'exclude_intervals': exclude_intervals, 'raw_variant_items': locus.get('rawVariantItems'), }) - exclude.pop('rawItems', None) def get_search_genes(gene_ids, genome_version): diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 5e94f6f2b7..8681ac739d 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -865,7 +865,7 @@ def test_variant_id_search(self): [],locus={'rawVariantItems': VARIANT_IDS[1]}, ) - @mock.patch('seqr.views.apis.variant_search_ap.MAX_EXPORT_VARIANTS', 2) + @mock.patch('seqr.views.apis.variant_search_api.MAX_EXPORT_VARIANTS', 2) @mock.patch('clickhouse_search.search.MAX_GENES_FOR_FILTER', 2) @mock.patch('clickhouse_search.search.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) @mock.patch('clickhouse_search.search.MAX_VARIANTS', 3) From 65b898c28c6937a61246a264643af5d98ad64c4b Mon Sep 17 00:00:00 2001 From: hanars Date: Wed, 29 Apr 2026 10:57:48 -0400 Subject: [PATCH 050/119] PR feedback Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- seqr/views/apis/variant_search_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 7e94b63f6e..76564c5d00 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -82,7 +82,7 @@ def query_variants_handler(request, search_hash): def _get_variants_with_cache(get_cache_key, get_variants, *args, **kwargs): cache_key = get_cache_key(*args, **kwargs) variants = safe_redis_get_json(cache_key) - if not variants: + if variants is None: variants = get_variants(*args, **kwargs) safe_redis_set_json(cache_key, variants, expire=timedelta(weeks=2)) return variants From 4e00232cab1d22f6b1113a69347cb4bda6a5ca93 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 29 Apr 2026 15:14:10 -0400 Subject: [PATCH 051/119] fix pedigree drawing with placeholder spouses --- .../LazyPedigreeImagePanel.jsx | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/ui/shared/components/panel/view-pedigree-image/LazyPedigreeImagePanel.jsx b/ui/shared/components/panel/view-pedigree-image/LazyPedigreeImagePanel.jsx index 5662693c92..3e445935a3 100644 --- a/ui/shared/components/panel/view-pedigree-image/LazyPedigreeImagePanel.jsx +++ b/ui/shared/components/panel/view-pedigree-image/LazyPedigreeImagePanel.jsx @@ -201,26 +201,24 @@ class BasePedigreeImage extends React.PureComponent { dataset.filter( ({ mother, father }) => mother || father, ).forEach((indiv) => { - if (indiv.mother && indiv.father) { - // pedigree js incorrectly considers people whose parents are different generations consanguineous - const mother = nameMap[indiv.mother] - const father = nameMap[indiv.father] - let missingGenParent = null - if (mother.top_level && !father.top_level) { - missingGenParent = mother - } else if (father.top_level && !mother.top_level) { - missingGenParent = father - } - if (missingGenParent) { - missingGenParent.top_level = false - this.addParent(`${missingGenParent.name}-father-placeholder`, missingGenParent, newParents, true) - this.addParent(`${missingGenParent.name}-mother-placeholder`, missingGenParent, newParents, false) - } - return + if (!indiv.mother || !indiv.father) { + const placeholderId = `${indiv.mother || indiv.father}-spouse-placeholder` + this.addParent(placeholderId, indiv, newParents, !indiv.father) + } + // pedigree js incorrectly considers people whose parents are different generations consanguineous + const mother = nameMap[indiv.mother] || newParents[indiv.mother] + const father = nameMap[indiv.father] || newParents[indiv.father] + let missingGenParent = null + if (mother.top_level && !father.top_level) { + missingGenParent = mother + } else if (father.top_level && !mother.top_level) { + missingGenParent = father + } + if (missingGenParent) { + missingGenParent.top_level = false + this.addParent(`${missingGenParent.name}-father-placeholder`, missingGenParent, newParents, true) + this.addParent(`${missingGenParent.name}-mother-placeholder`, missingGenParent, newParents, false) } - - const placeholderId = `${indiv.mother || indiv.father}-spouse-placeholder` - this.addParent(placeholderId, indiv, newParents, !indiv.father) }) return this.yobToAge([...dataset, ...Object.values(newParents)]) From c1348bde2981ee971fc849641f33b2ac43700231 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 29 Apr 2026 16:29:45 -0400 Subject: [PATCH 052/119] actually send sear ch hash in tag creation body --- ui/pages/Search/components/VariantSearchResults.jsx | 6 +++--- ui/redux/rootReducer.js | 4 ++-- ui/shared/components/panel/variants/FamilyVariantTags.jsx | 8 ++++++-- ui/shared/components/panel/variants/Variants.jsx | 4 +++- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ui/pages/Search/components/VariantSearchResults.jsx b/ui/pages/Search/components/VariantSearchResults.jsx index ab86291822..8ce1167fe0 100644 --- a/ui/pages/Search/components/VariantSearchResults.jsx +++ b/ui/pages/Search/components/VariantSearchResults.jsx @@ -43,10 +43,10 @@ const LargeRow = styled(Grid.Row)` const scrollToTop = () => window.scrollTo(0, 0) -const DisplayVariants = React.memo(({ displayVariants, compoundHetToggle }) => ( +const DisplayVariants = React.memo(({ displayVariants, ...props }) => ( - + )) @@ -95,7 +95,7 @@ const BaseVariantSearchResultsContent = React.memo(({ , - , + , diff --git a/ui/redux/rootReducer.js b/ui/redux/rootReducer.js index 99bf5f2136..3034180c7c 100644 --- a/ui/redux/rootReducer.js +++ b/ui/redux/rootReducer.js @@ -204,12 +204,12 @@ const navigateSavedHashedSearchHelper = (resultsPath, hashKey) => (search, navig export const navigateSavedHashedSearch = navigateSavedHashedSearchHelper('results', 'searchesByHash') export const navigateFamiliesSearch = navigateSavedHashedSearchHelper('families', 'searchFamiliesByHash') -const updateSavedVariant = (values, action = 'create') => (dispatch, getState) => new HttpRequestHelper( +const updateSavedVariant = (values, action = 'create') => dispatch => new HttpRequestHelper( `/api/saved_variant/${action}`, (responseJson) => { dispatch({ type: RECEIVE_DATA, updatesById: responseJson }) }, -).post({ searchHash: getState().currentSearchHash, ...values }) +).post(values) export const updateVariantNote = (values) => { if (values.variantGuids) { diff --git a/ui/shared/components/panel/variants/FamilyVariantTags.jsx b/ui/shared/components/panel/variants/FamilyVariantTags.jsx index 2d53cc6f41..341421613f 100644 --- a/ui/shared/components/panel/variants/FamilyVariantTags.jsx +++ b/ui/shared/components/panel/variants/FamilyVariantTags.jsx @@ -393,10 +393,14 @@ const mapStateToProps = (state, ownProps) => { const mapDispatchToProps = (dispatch, ownProps) => ({ dispatchUpdateVariantNote: updates => dispatch( - updateVariantNote({ ...updates, variant: ownProps.variant, familyGuid: ownProps.familyGuid }), + updateVariantNote({ + ...updates, variant: ownProps.variant, searchHash: ownProps.searchHash, familyGuid: ownProps.familyGuid, + }), ), dispatchUpdateFamilyVariantTags: updates => dispatch( - updateVariantTags({ ...updates, variant: ownProps.variant, familyGuid: ownProps.familyGuid }), + updateVariantTags({ + ...updates, variant: ownProps.variant, searchHash: ownProps.searchHash, familyGuid: ownProps.familyGuid, + }), ), dispatchUpdateFamilyVariantFunctionalTags: updates => dispatch( updateVariantTags({ ...updates, variant: ownProps.variant, familyGuid: ownProps.familyGuid }, 'functional_data'), diff --git a/ui/shared/components/panel/variants/Variants.jsx b/ui/shared/components/panel/variants/Variants.jsx index 7bad475092..bd16675dfa 100644 --- a/ui/shared/components/panel/variants/Variants.jsx +++ b/ui/shared/components/panel/variants/Variants.jsx @@ -78,7 +78,7 @@ const tagFamily = tag => ( const VariantLayout = ( { variant, compoundHetToggle, mainGeneId, isCompoundHet, linkToSavedVariants, topContent, - bottomContent, children, ...rowProps + bottomContent, children, searchHash, ...rowProps }, ) => { const coreVariant = Array.isArray(variant) ? variant[0] : variant @@ -95,6 +95,7 @@ const VariantLayout = ( variant={variant} isCompoundHet={isCompoundHet} linkToSavedVariants={linkToSavedVariants} + searchHash={searchHash} /> @@ -131,6 +132,7 @@ VariantLayout.propTypes = { topContent: PropTypes.node, bottomContent: PropTypes.node, children: PropTypes.node, + searchHash: PropTypes.string, } export const Variant = React.memo(( From 976046bd9f989a382b680ccf939a00c9b7cc4f8d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 30 Apr 2026 12:49:49 -0400 Subject: [PATCH 053/119] add sequencing type to rna samples --- .../0088_rnasample_sequencing_type.py | 19 +++++++++++++++++++ seqr/models.py | 7 +++++++ 2 files changed, 26 insertions(+) create mode 100644 seqr/migrations/0088_rnasample_sequencing_type.py diff --git a/seqr/migrations/0088_rnasample_sequencing_type.py b/seqr/migrations/0088_rnasample_sequencing_type.py new file mode 100644 index 0000000000..dddada36e7 --- /dev/null +++ b/seqr/migrations/0088_rnasample_sequencing_type.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.24 on 2026-04-30 16:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('seqr', '0087_dataset'), + ] + + operations = [ + migrations.AddField( + model_name='rnasample', + name='sequencing_type', + field=models.CharField(choices=[('K', 'Kinnex'), ('T', 'Tru-Seq'), ('W', 'Watchmaker')], default='T', max_length=1), + preserve_default=False, + ), + ] diff --git a/seqr/models.py b/seqr/models.py index 7942ec7bf4..b2f53553cf 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -754,10 +754,17 @@ class RnaSample(ModelWithGUID): ('S', 'skin'), ) + SEQUENCING_CHOICES = ( + ('K', 'Kinnex'), + ('T', 'Tru-Seq'), + ('W', 'Watchmaker'), + ) + individual = models.ForeignKey('Individual', on_delete=models.PROTECT) data_type = models.CharField(max_length=1, choices=DATA_TYPE_CHOICES) tissue_type = models.CharField(max_length=2, choices=TISSUE_TYPE_CHOICES) + sequencing_type = models.CharField(max_length=1, choices=SEQUENCING_CHOICES) data_source = models.TextField() is_active = models.BooleanField(default=False) From 8f0360fa5ab8ac5ba7338aabe3350d0661d2e73e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 30 Apr 2026 13:05:50 -0400 Subject: [PATCH 054/119] return sequencing type to client --- seqr/views/apis/family_api.py | 19 ++++++++++--------- seqr/views/utils/orm_to_json_utils.py | 1 + .../Project/components/RnaSeqResultPage.jsx | 1 + ui/pages/Project/selectors.js | 1 + .../components/panel/variants/RnaSeqTpm.jsx | 1 + .../table/RnaSeqJunctionOutliersTable.jsx | 1 + 6 files changed, 15 insertions(+), 9 deletions(-) diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 0b934912cf..7939a5fe9a 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -488,24 +488,25 @@ def get_family_rna_seq_data(request, family_guid, gene_id): family = Family.objects.get(guid=family_guid) check_project_permissions(family.project, request.user) - response = defaultdict(lambda: {'individualData': {}}) + response = defaultdict(lambda: defaultdict(lambda: {'individualData': {}})) tpm_data = RnaSeqTpm.objects.filter( gene_id=gene_id, sample__individual__family=family).prefetch_related('sample', 'sample__individual') for tpm in tpm_data: indiv = tpm.sample.individual - response[tpm.sample.tissue_type]['individualData'][indiv.display_name or indiv.individual_id] = tpm.tpm + response[tpm.sample.tissue_type][tpm.sample.sequencing_type]['individualData'][indiv.display_name or indiv.individual_id] = tpm.tpm project_guids = get_project_guids_user_can_view(request.user) internal_projects = get_internal_projects() if anvil_enabled() else None for tissue in response.keys(): - tpms = RnaSeqTpm.objects.filter(sample__tissue_type=tissue, gene_id=gene_id) - response[tissue]['myData'] = list(tpms.filter( - sample__individual__family__project__guid__in=project_guids, - ).order_by('tpm').values_list('tpm', flat=True)) - if internal_projects is not None: - response[tissue]['rdgData'] = list(tpms.filter( - sample__individual__family__project__in=internal_projects, + for sequencing_type in response[tissue].keys(): + tpms = RnaSeqTpm.objects.filter(sample__tissue_type=tissue, sequencing_type=sequencing_type, gene_id=gene_id) + response[tissue][sequencing_type]['myData'] = list(tpms.filter( + sample__individual__family__project__guid__in=project_guids, ).order_by('tpm').values_list('tpm', flat=True)) + if internal_projects is not None: + response[tissue][sequencing_type]['rdgData'] = list(tpms.filter( + sample__individual__family__project__in=internal_projects, + ).order_by('tpm').values_list('tpm', flat=True)) return create_json_response(response) diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 01513afab9..cd4cab9d9c 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -788,6 +788,7 @@ def get_json_for_rna_seq_outliers(filters, significant_only=True, individual_gui models, nested_fields=[ {'fields': ('sample', 'tissue_type'), 'key': 'tissueType'}, + {'fields': ('sample', 'sequencing_type'), 'key': 'sequencingType'}, {'fields': ('sample', 'individual', 'guid'), 'key': 'individualGuid', 'value': individual_guid}, ], additional_values={'isSignificant': Value(True)} if significant_only else { diff --git a/ui/pages/Project/components/RnaSeqResultPage.jsx b/ui/pages/Project/components/RnaSeqResultPage.jsx index 8477420bb2..57d43210f8 100644 --- a/ui/pages/Project/components/RnaSeqResultPage.jsx +++ b/ui/pages/Project/components/RnaSeqResultPage.jsx @@ -50,6 +50,7 @@ class BaseRnaSeqResultPage extends React.PureComponent { } onTissueChange = (e, data) => { + // TODO this.setState({ tissueType: data.value }) } diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index 9d533feb31..e6bc8e0f4b 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -852,6 +852,7 @@ export const getIndividualPhenotypeGeneScores = createSelector( export const getTissueOptionsByIndividualGuid = createSelector( getRnaSeqDataByIndividual, (rnaSeqDataByIndividualGuid) => { + // TODO const tissueTypesByIndividualGuid = Object.entries(rnaSeqDataByIndividualGuid || {}).map( ([individualGuid, rnaSeqData]) => ([ individualGuid, diff --git a/ui/shared/components/panel/variants/RnaSeqTpm.jsx b/ui/shared/components/panel/variants/RnaSeqTpm.jsx index fe1612a680..e68abf3ff2 100644 --- a/ui/shared/components/panel/variants/RnaSeqTpm.jsx +++ b/ui/shared/components/panel/variants/RnaSeqTpm.jsx @@ -189,6 +189,7 @@ const renderGtex = (gtexExpressionData, familyExpressionData, containerElement) const gtexByTissue = ((gtexExpressionData || {}).data || []).reduce((acc, { data, tissueSiteDetailId }) => ({ ...acc, [GTEX_TISSUE_LOOKUP[tissueSiteDetailId]]: data, }), {}) + // TODO const boxplotData = Object.entries(familyExpressionData).reduce(( acc, [tissue, { rdgData, myData, individualData }], ) => ([ diff --git a/ui/shared/components/table/RnaSeqJunctionOutliersTable.jsx b/ui/shared/components/table/RnaSeqJunctionOutliersTable.jsx index bd0babdcd4..04e231cf45 100644 --- a/ui/shared/components/table/RnaSeqJunctionOutliersTable.jsx +++ b/ui/shared/components/table/RnaSeqJunctionOutliersTable.jsx @@ -15,6 +15,7 @@ const RNA_SEQ_SPLICE_NUM_FIELDS = ['pValue', 'pAdjust', 'deltaIntronJaccardIndex const RNA_SEQ_SPLICE_DETAIL_FIELDS = ['type'] const openReads = (updateReads, row) => () => { + // TODO const { chrom, start, end, tissueType, familyGuid } = row updateReads(familyGuid, getLocus(chrom, start, RNASEQ_JUNCTION_PADDING, end - start), [JUNCTION_TYPE, COVERAGE_TYPE], tissueType) From e08b116bd29f51b615054888c8a6c474be17e992 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 30 Apr 2026 13:59:57 -0400 Subject: [PATCH 055/119] first pass airtable sequencing type --- seqr/views/apis/data_manager_api.py | 11 ++++++++++- seqr/views/utils/dataset_utils.py | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 4e1b386b21..615ebf19f7 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -30,6 +30,7 @@ logger = SeqrLogger(__name__) RNA = 'RNA' +SEQUENCING_FIELD = 'RNASequencingProduct' TISSUE_FIELD = 'TissueOfOrigin' AIRTABLE_TISSUE_TYPE_MAP = { 'whole_blood': 'Blood', @@ -53,7 +54,7 @@ def update_rna_seq(request): airtable_samples = _get_dataset_type_samples_for_matched_pdos( ['RNA ready to load'], request.user, RNA, None, allow_invalid_pdos=True, sample_fields=[ TISSUE_FIELD, 'WESSampleID_RNAMapping', 'WGSSampleID_RNAMapping', - ], + ], pdo_fields=[SEQUENCING_FIELD], ) sample_metadata_mapping = {} misconfigured_samples = defaultdict(list) @@ -77,12 +78,20 @@ def update_rna_seq(request): elif not pdos or pdos[0]['project_guid'] is None: error_messages.append('no project specified') + airtable_sequencing_type = pdos[0].get(SEQUENCING_FIELD) + if not airtable_sequencing_type: + error_messages.append('no sequencing product specified') + sequencing_type = next((st for st, label in RnaSample.SEQUENCING_CHOICES if airtable_sequencing_type.startswith(st)), None) + if not sequencing_type: + error_messages.append('invalid sequencing product specified') + if error_messages: for sample_id in sample_ids: misconfigured_samples[sample_id].extend(error_messages) else: metadata = { 'tissue': TISSUE_TYPE_MAP[sample[TISSUE_FIELD][0]], + 'sequencing_type': sequencing_type, 'project_guid': sample['pdos'][0]['project_guid'], 'sample_id': sample.get('CollaboratorSampleID') or sample['sample_id'], } diff --git a/seqr/views/utils/dataset_utils.py b/seqr/views/utils/dataset_utils.py index 5a3714f326..4ab34da6ff 100644 --- a/seqr/views/utils/dataset_utils.py +++ b/seqr/views/utils/dataset_utils.py @@ -317,6 +317,7 @@ def load_rna_seq(request_json, user, **kwargs): data_type = request_json['dataType'] file_path = request_json['file'] + # TODO sequencing_type try: sample_guids, file_name_prefix, info, warnings = _load_rna_seq( data_type, file_path, user, **kwargs, From 44176f96dfae90255076ae6163ca534bf0db673a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 30 Apr 2026 15:13:00 -0400 Subject: [PATCH 056/119] fix airtable lookup --- seqr/views/apis/data_manager_api.py | 12 ++++++------ seqr/views/utils/airtable_utils.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 615ebf19f7..7803d8d2c5 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -73,17 +73,17 @@ def update_rna_seq(request): error_messages.append('invalid tissue specified') pdos = sample.get('pdos', []) + sequencing_type = None if len(pdos) > 1: error_messages.append('multiple conflicting PDOs') elif not pdos or pdos[0]['project_guid'] is None: error_messages.append('no project specified') - - airtable_sequencing_type = pdos[0].get(SEQUENCING_FIELD) - if not airtable_sequencing_type: + elif not pdos[0][SEQUENCING_FIELD]: error_messages.append('no sequencing product specified') - sequencing_type = next((st for st, label in RnaSample.SEQUENCING_CHOICES if airtable_sequencing_type.startswith(st)), None) - if not sequencing_type: - error_messages.append('invalid sequencing product specified') + else: + sequencing_type = next((st for st, label in RnaSample.SEQUENCING_CHOICES if pdos[0][SEQUENCING_FIELD].startswith(st)), None) + if not sequencing_type: + error_messages.append('invalid sequencing product specified') if error_messages: for sample_id in sample_ids: diff --git a/seqr/views/utils/airtable_utils.py b/seqr/views/utils/airtable_utils.py index ca806afe0f..3d57e5d5f3 100644 --- a/seqr/views/utils/airtable_utils.py +++ b/seqr/views/utils/airtable_utils.py @@ -207,7 +207,7 @@ def get_samples_for_matched_pdos(self, pdo_statuses, pdo_fields=None, sample_fie project_guids = [match and match.group(1) for match in project_matches] pdos = [{ 'project_guid': project_guids[i] if len(project_guids) > 1 else project_guids[0], - **{field: sample[field][i] for field in pdo_fields} + **{field: sample[field][i] if len(sample[field]) == len(sample['PDOStatus']) else None for field in pdo_fields} } for i, status in enumerate(sample['PDOStatus']) if status in pdo_statuses] if project_guid: pdos = [pdo for pdo in pdos if pdo['project_guid'] == project_guid] From 9acf1630e8c5a2c7e06b4b09bc43d99b0bed2fcb Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 1 May 2026 10:37:09 -0400 Subject: [PATCH 057/119] actually create samples with sequencing type --- seqr/views/utils/dataset_utils.py | 22 +++++++++---------- .../Project/components/EditDatasetsButton.jsx | 11 ++++++++++ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/seqr/views/utils/dataset_utils.py b/seqr/views/utils/dataset_utils.py index 4ab34da6ff..90071f8bae 100644 --- a/seqr/views/utils/dataset_utils.py +++ b/seqr/views/utils/dataset_utils.py @@ -23,8 +23,8 @@ def _create_rna_samples(sample_data, sample_guid_ids_to_load, user, **kwargs): new_samples = [ - RnaSample(individual_id=individual_id, tissue_type=tissue_type, **kwargs) - for individual_id, tissue_type in sample_data.items() + RnaSample(individual_id=individual_id, tissue_type=tissue_type, sequencing_type=sequencing_type, **kwargs) + for individual_id, (tissue_type, sequencing_type) in sample_data.items() ] new_sample_models = RnaSample.bulk_create(user, new_samples) new_sample_ids = [s.id for s in new_sample_models] @@ -34,7 +34,7 @@ def _create_rna_samples(sample_data, sample_guid_ids_to_load, user, **kwargs): def _get_rna_sample_data_by_key(values=None, **kwargs): - key_fields = ['individual_id', 'tissue_type'] + key_fields = ['individual_id', 'tissue_type', 'sequencing_type'] return { tuple(s.pop(k) for k in key_fields): s for s in RnaSample.objects.filter(**kwargs).values('guid', *key_fields, **(values or {})) @@ -191,7 +191,7 @@ def _load_rna_seq_file( ) inactivate_samples = { sample['guid']: key[0] for key, sample in potential_inactivate_samples_by_key.items() - if key in set(samples_to_create.items()) and sample['guid'] not in loaded_samples + if key in {tuple([k, *v]) for k, v in samples_to_create.items()} and sample['guid'] not in loaded_samples } errors, warnings = _process_rna_errors( @@ -223,7 +223,8 @@ def _parse_rna_row(sample_id, row_dict, potential_samples, loaded_samples, gene_ return tissue_type = individual['tissue'] - potential_sample = potential_samples.get((individual['id'], tissue_type)) + sequencing_type = individual['sequencing_type'] + potential_sample = potential_samples.get((individual['id'], tissue_type, sequencing_type)) if (potential_sample or {}).get('active'): loaded_samples.add(potential_sample['guid']) return @@ -231,7 +232,7 @@ def _parse_rna_row(sample_id, row_dict, potential_samples, loaded_samples, gene_ if potential_sample: sample_guid_ids_to_load[potential_sample['guid']] = sample_id else: - samples_to_create[individual['id']] = tissue_type + samples_to_create[individual['id']] = (tissue_type, sequencing_type) if has_errors: # If there are definite errors, do not process/save data, just continue to check for additional errors @@ -317,10 +318,9 @@ def load_rna_seq(request_json, user, **kwargs): data_type = request_json['dataType'] file_path = request_json['file'] - # TODO sequencing_type try: sample_guids, file_name_prefix, info, warnings = _load_rna_seq( - data_type, file_path, user, **kwargs, + data_type, file_path, user, **kwargs, sequencing_type=request_json.get('sequencingType'), tissue=request_json.get('tissue'), ignore_extra_samples=request_json.get('ignoreExtraSamples'), skip_new_sample_validation=request_json.get('skipNewSampleValidation'), ) @@ -337,7 +337,7 @@ def load_rna_seq(request_json, user, **kwargs): }, 200 -def _load_rna_seq(data_type, file_path, user, sample_metadata_mapping=None, project_guid=None, tissue=None, **kwargs): +def _load_rna_seq(data_type, file_path, user, sample_metadata_mapping=None, project_guid=None, sequencing_type=None, tissue=None, **kwargs): config = RNA_DATA_TYPE_CONFIGS[data_type] model_cls = config['model_class'] file_name = file_path.split('/')[-1] @@ -350,7 +350,7 @@ def _load_rna_seq(data_type, file_path, user, sample_metadata_mapping=None, proj individuals = Individual.objects.filter(family__project__in=projects) individual_data_by_id = _get_individual_metadata_mapping(sample_metadata_mapping, individuals) if sample_metadata_mapping else { - i['individual_id']: {**i, 'tissue': tissue} for i in individuals.values('id', 'individual_id') + i['individual_id']: {**i, 'tissue': tissue, 'sequencing_type': sequencing_type} for i in individuals.values('id', 'individual_id') } potential_samples = _get_rna_sample_data_by_key( individual_id__in={i['id'] for i in individual_data_by_id.values()}, @@ -416,7 +416,7 @@ def _get_individual_metadata_mapping(sample_metadata_mapping, individuals): individual_id = indiv['individual_id'] sample_metadata = sample_metadata_mapping[individual_id] if indiv.pop('family__project__guid') == sample_metadata['project_guid']: - individual_data = {**indiv, 'tissue': sample_metadata['tissue']} + individual_data = {**indiv, 'tissue': sample_metadata['tissue'], 'sequencing_type': sample_metadata['sequencing_type']} individual_data_by_id[individual_id] = individual_data # Support when data is provided using either the raw sample ID or the already mapped seqr ID if individual_id != sample_metadata['sample_id']: diff --git a/ui/pages/Project/components/EditDatasetsButton.jsx b/ui/pages/Project/components/EditDatasetsButton.jsx index 90e5e2fb61..da6b3ed99f 100644 --- a/ui/pages/Project/components/EditDatasetsButton.jsx +++ b/ui/pages/Project/components/EditDatasetsButton.jsx @@ -72,6 +72,17 @@ const PROJECT_LOAD_RNA_FIELDS = [ options: Object.entries(TISSUE_DISPLAY).map(([value, name]) => ({ value, name })), validate: validators.required, }, + { + name: 'sequencingType', + label: 'Sequencing Type', + component: Select, + options: [ + { value: 'K', name: 'Kinnex' }, + { value: 'T', name: 'TruSeq' }, + { value: 'W', name: 'Watchmaker' }, + ], + validate: validators.required, + }, ...LOAD_RNA_FIELDS.slice(-2), ] From d1d55442435909617eda2a5b85091da21ffa0076 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 1 May 2026 10:57:44 -0400 Subject: [PATCH 058/119] fix load project data tests --- seqr/fixtures/1kg_project.json | 6 ++++++ seqr/views/apis/project_api_tests.py | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index 68a415bcdc..995e6eb076 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -951,6 +951,7 @@ "individual": 19, "is_active": true, "tissue_type": "F", + "sequencing_type": "W", "data_source": "fibs_samples.tsv.gz", "data_type": "S" } @@ -1056,6 +1057,7 @@ "individual": 1, "is_active": true, "tissue_type": "F", + "sequencing_type": "W", "data_source": "muscle_samples.tsv.gz", "data_type": "S" } @@ -1071,6 +1073,7 @@ "individual": 1, "is_active": true, "tissue_type": "M", + "sequencing_type": "T", "data_source": "muscle_samples.tsv.gz", "data_type": "S" } @@ -1101,6 +1104,7 @@ "individual": 1, "is_active": true, "tissue_type": "F", + "sequencing_type": "T", "data_source": "muscle_samples.tsv.gz", "data_type": "T" } @@ -1116,6 +1120,7 @@ "individual": 1, "is_active": true, "tissue_type": "M", + "sequencing_type": "T", "data_source": "muscle_samples.tsv.gz", "data_type": "T" } @@ -1131,6 +1136,7 @@ "individual": 1, "is_active": true, "tissue_type": "M", + "sequencing_type": "T", "data_source": "muscle_samples.tsv.gz", "data_type": "E" } diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index 96b5538d38..4155a9b96a 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -704,7 +704,7 @@ def test_update_project_rna_tpm(self): self._test_update_project_rna('T', **RNA_DATA_TYPE_PARAMS['T'], single_sample_file=True) def test_update_project_rna_splice_outlier(self): - self._test_update_project_rna('S', **RNA_DATA_TYPE_PARAMS['S'], tissue='F', allow_missing_gene=True) + self._test_update_project_rna('S', **RNA_DATA_TYPE_PARAMS['S'], tissue='F', sequencing_type='W', allow_missing_gene=True) @mock.patch('seqr.utils.communication_utils.BASE_URL', 'https://test-seqr.org/') @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') @@ -719,7 +719,7 @@ def test_update_project_rna_splice_outlier(self): def _test_update_project_rna(self, data_type, mock_subprocess, mock_does_file_exist, mock_open, mock_os, mock_send_slack, mock_send_email, mock_datetime, sample_guid=None, model_cls=None, rows=None, parsed_file_data=None, required_columns=None, allow_missing_gene=False, - tissue='M', message_data_type=None, single_sample_file=False, **kwargs): + tissue='M', sequencing_type='T', message_data_type=None, single_sample_file=False, **kwargs): mock_datetime.now.return_value = datetime(2025, 4, 15) mock_os.path.join.side_effect = lambda *args: '/'.join(args) initial_model_count = model_cls.objects.count() @@ -731,7 +731,7 @@ def _test_update_project_rna(self, data_type, mock_subprocess, mock_does_file_ex # Test errors file = f'{self.TEMP_DIR}/new_samples.tsv.gz' - body = {'dataType': data_type, 'file': file, 'tissue': tissue} + body = {'dataType': data_type, 'file': file, 'tissue': tissue, 'sequencingType': sequencing_type} self._set_file_not_found(file, mock_subprocess, mock_does_file_exist, mock_open) self.reset_logs() response = self.client.post(url, content_type='application/json', data=json.dumps(body)) @@ -786,7 +786,7 @@ def _test_update_project_rna(self, data_type, mock_subprocess, mock_does_file_ex # test database models are correct self.assertEqual(model_cls.objects.count(), initial_model_count - initial_sample_model_count) rna_samples = RnaSample.objects.filter( - tissue_type=tissue, data_type=data_type, data_source='new_samples.tsv.gz', is_active=False, + tissue_type=tissue, data_type=data_type, data_source='new_samples.tsv.gz', is_active=False, sequencing_type=sequencing_type, ) self.assertEqual(rna_samples.count(), 1 if single_sample_file else 2) guid_map = {'NA19675_1': rna_samples.get(individual_id=1).guid} From ddbc6fe1d4ad5c65188c0aa0ce14d0333fea51ac Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 1 May 2026 11:40:08 -0400 Subject: [PATCH 059/119] fix airtable dfata laoding tests --- seqr/views/apis/data_manager_api_tests.py | 27 +++++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 00d6500c25..b40b2a37d6 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -252,6 +252,7 @@ 'CollaboratorSampleID': 'NA19675_D2', 'SeqrProject': ['https://seqr.broadinstitute.org/project/R0001_1kg/project_page'], 'PDOStatus': ['RNA ready to load'], + 'RNASequencingProduct': ['Tru-Seq polyA 75M'], } }, { @@ -264,6 +265,7 @@ ], 'PDOStatus': ['RNA ready to load', 'Available in seqr'], 'TissueOfOrigin': ['Muscle'], + 'RNASequencingProduct': ['Tru-Seq polyA 75M', 'Kinnex long read total RNA'], } }, { @@ -273,6 +275,7 @@ 'SeqrProject': ['https://seqr.broadinstitute.org/project/R0003_test/project_page'], 'PDOStatus': ['RNA ready to load'], 'TissueOfOrigin': ['Muscle'], + 'RNASequencingProduct': ['Tru-Seq polyA 75M'], } }, { @@ -285,6 +288,7 @@ ], 'PDOStatus': ['RNA ready to load', 'RNA ready to load'], 'TissueOfOrigin': ['Muscle', 'Brain'], + 'RNASequencingProduct': ['Tru-Seq polyA 75M', 'Watchmaker total RNA 200M'], } }, { @@ -295,6 +299,7 @@ 'https://seqr.broadinstitute.org/project/R0002_empty/project_page', ], 'PDOStatus': ['RNA ready to load'], + 'RNASequencingProduct': ['Tru-Seq polyA 75M', 'Watchmaker total RNA 200M'], } }, { @@ -303,10 +308,10 @@ 'CollaboratorSampleID': 'NA12347', 'SeqrProject': [ 'https://seqr.broadinstitute.org/project/R0002_empty/project_page', - 'https://seqr.broadinstitute.org/project/R0004_non_analyst_project/project_page', ], - 'PDOStatus': ['RNA ready to load', 'RNA ready to load'], + 'PDOStatus': ['RNA ready to load'], 'TissueOfOrigin': ['Muscle'], + 'RNASequencingProduct': ['Custom RNA Product'], } }, { @@ -316,6 +321,7 @@ 'SeqrProject': ['R0003_test'], 'PDOStatus': ['RNA ready to load'], 'TissueOfOrigin': ['Custom Tissue'], + 'RNASequencingProduct': ['Custom RNA Product'], } }, { @@ -325,6 +331,7 @@ 'SeqrProject': ['https://seqr.broadinstitute.org/project/R0003_test/project_page'], 'PDOStatus': ['RNA ready to load'], 'TissueOfOrigin': ['Muscle'], + 'RNASequencingProduct': ['Tru-Seq polyA 75M'], } }, *INVALID_AIRTABLE_SAMPLE_RECORDS['records'], @@ -488,6 +495,7 @@ def test_luigi_proxy(self, mock_hostname): ], 'skipped_samples': 'NA19675_D3, NA20878', 'sample_tissue_type': 'Fibroblast', + 'sample_sequencing_type': 'Watchmaker total RNA 200M', 'num_parsed_samples': 4, 'initial_model_count': 7, 'deleted_count': 4, @@ -511,10 +519,11 @@ def _has_expected_file_loading_logs(self, file, file_path, user, info=None, warn self.assert_json_logs(user, expected_logs) - def _check_rna_sample_model(self, individual_id, data_source, data_type, tissue_type, is_active_sample=True): + def _check_rna_sample_model(self, individual_id, data_source, data_type, tissue_type, sequencing_type='T', is_active_sample=True): tissue_type = tissue_type[0] + sequencing_type = sequencing_type[0] rna_samples = RnaSample.objects.filter( - individual_id=individual_id, tissue_type=tissue_type, data_source=data_source, data_type=data_type, + individual_id=individual_id, tissue_type=tissue_type, sequencing_type=sequencing_type, data_source=data_source, data_type=data_type, ) self.assertEqual(len(rna_samples), 1) sample = rna_samples.first() @@ -601,12 +610,14 @@ def _set_gzip_file_iter_stdout(mock_file, rows): self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'errors': [f'Unable to load the following samples with no match: {loaded_data_row[0]}'], 'warnings': None}) + sequencing_type = params.get('sample_sequencing_type', AIRTABLE_RNA_SAMPLE_RECORDS[0]['fields']['RNASequencingProduct'][0]) airtable_sample_records = [ { **AIRTABLE_RNA_SAMPLE_RECORDS[0], 'fields': { **AIRTABLE_RNA_SAMPLE_RECORDS[0]['fields'], 'TissueOfOrigin': [params['sample_tissue_type']], + 'RNASequencingProduct': [sequencing_type], } }, *AIRTABLE_RNA_SAMPLE_RECORDS[1:], @@ -633,10 +644,12 @@ def _set_gzip_file_iter_stdout(mock_file, rows): self.assertEqual(response.status_code, 400) self.assertListEqual(response.json()['errors'], [ 'Unable to load the following samples from Airtable with no corresponding seqr ID: NA11111', + 'Unable to load the following samples that are improperly configured in Airtable with invalid sequencing product specified: NA12347', 'Unable to load the following samples that are improperly configured in Airtable with invalid tissue specified: NA12348', - 'Unable to load the following samples that are improperly configured in Airtable with multiple conflicting PDOs: NA12345, NA12347', + 'Unable to load the following samples that are improperly configured in Airtable with multiple conflicting PDOs: NA12345', 'Unable to load the following samples that are improperly configured in Airtable with multiple tissues specified: NA12345', 'Unable to load the following samples that are improperly configured in Airtable with no project specified: NA12348', + 'Unable to load the following samples that are improperly configured in Airtable with no sequencing product specified: NA12346', 'Unable to load the following samples that are improperly configured in Airtable with no tissue specified: NA12346', ]) @@ -700,7 +713,7 @@ def _test_basic_data_loading(data, num_parsed_samples, num_loaded_samples, new_s self.assert_expected_airtable_call( call_index=0, filter_formula="AND(LEN({PassingCollaboratorSampleIDs})>0,OR(SEARCH('RNA ready to load',ARRAYJOIN(PDOStatus,';'))))", - fields=['CollaboratorSampleID', 'SeqrCollaboratorSampleID', 'PDOStatus', 'SeqrProject', 'TissueOfOrigin', 'WESSampleID_RNAMapping', 'WGSSampleID_RNAMapping'], + fields=['CollaboratorSampleID', 'SeqrCollaboratorSampleID', 'PDOStatus', 'SeqrProject', 'RNASequencingProduct', 'TissueOfOrigin', 'WESSampleID_RNAMapping', 'WGSSampleID_RNAMapping'], ) return response_json, new_sample_guid @@ -752,7 +765,7 @@ def _test_basic_data_loading(data, num_parsed_samples, num_loaded_samples, new_s # test database models are correct self.assertEqual(model_cls.objects.count(), params['initial_model_count'] - deleted_count) sample_guid = self._check_rna_sample_model(individual_id=1, data_source='new_muscle_samples.tsv.gz', data_type=data_type, - tissue_type=params.get('sample_tissue_type'), is_active_sample=False) + tissue_type=params.get('sample_tissue_type'), sequencing_type=sequencing_type, is_active_sample=False) self.assertSetEqual(set(response_json['sampleGuids']), {sample_guid, new_sample_guid}) # test correct file interactions From 3e8f861b1031f96b4ece0aeb38c832272b2fec0f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 1 May 2026 11:46:33 -0400 Subject: [PATCH 060/119] fix tpm data tests --- seqr/fixtures/1kg_project.json | 2 +- seqr/views/apis/family_api.py | 2 +- seqr/views/apis/family_api_tests.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index 995e6eb076..33a5913f17 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -1104,7 +1104,7 @@ "individual": 1, "is_active": true, "tissue_type": "F", - "sequencing_type": "T", + "sequencing_type": "W", "data_source": "muscle_samples.tsv.gz", "data_type": "T" } diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index 7939a5fe9a..d6acdb0104 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -499,7 +499,7 @@ def get_family_rna_seq_data(request, family_guid, gene_id): internal_projects = get_internal_projects() if anvil_enabled() else None for tissue in response.keys(): for sequencing_type in response[tissue].keys(): - tpms = RnaSeqTpm.objects.filter(sample__tissue_type=tissue, sequencing_type=sequencing_type, gene_id=gene_id) + tpms = RnaSeqTpm.objects.filter(sample__tissue_type=tissue, sample__sequencing_type=sequencing_type, gene_id=gene_id) response[tissue][sequencing_type]['myData'] = list(tpms.filter( sample__individual__family__project__guid__in=project_guids, ).order_by('tpm').values_list('tpm', flat=True)) diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index 381a29e323..52ea9b1e1d 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -649,17 +649,17 @@ def test_get_family_rna_seq_data(self): self.assertEqual(response.status_code, 200) expected_response = { - 'F': {'individualData': {'NA19675_1': 1.01}, 'myData': [1.01]}, - 'M': {'individualData': {'NA19675_1': 8.38}, 'myData': [8.38]} + 'F': {'W': {'individualData': {'NA19675_1': 1.01}, 'myData': [1.01]}}, + 'M': {'T': {'individualData': {'NA19675_1': 8.38}, 'myData': [8.38]}} } if self.INCLUDE_RDG_TPMS: - expected_response = {k: {**v, 'rdgData': [*v['myData']]} for k, v in expected_response.items()} + expected_response = {tissue: {k: {**v, 'rdgData': [*v['myData']]} for k, v in data.items()} for tissue, data in expected_response.items()} self.assertDictEqual(response.json(), expected_response) self.login_manager() response = self.client.get(url) self.assertEqual(response.status_code, 200) - expected_response['F']['myData'].append(2.36) + expected_response['F']['W']['myData'].append(2.36) self.assertDictEqual(response.json(), expected_response) def test_get_family_phenotype_gene_scores(self): From b516a731dece0669474d8d0b9b605003ce533176 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 1 May 2026 11:48:49 -0400 Subject: [PATCH 061/119] fix tests --- seqr/views/apis/individual_api_tests.py | 8 ++++---- seqr/views/apis/saved_variant_api_tests.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 00a0bf7a0f..027b5a8237 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -1374,16 +1374,16 @@ def test_get_individual_rna_seq_data(self): 'ENSG00000135953': [{ 'geneId': 'ENSG00000135953', 'zScore': 7.31, 'pValue': 0.00000000000948, 'pAdjust': 0.00000000781, 'isSignificant': True, - 'tissueType': 'M', + 'tissueType': 'M', 'sequencingType': 'T', }], 'ENSG00000240361': [{ 'geneId': 'ENSG00000240361', 'zScore': -4.08, 'pValue': 5.88, 'pAdjust': 0.09, 'isSignificant': False, - 'tissueType': 'M', + 'tissueType': 'M', 'sequencingType': 'T', }], 'ENSG00000268903': [{ 'geneId': 'ENSG00000268903', 'zScore': 7.08, 'pValue':0.000000000588, 'pAdjust': 0.00000000139, 'isSignificant': True, - 'tissueType': 'M', + 'tissueType': 'M', 'sequencingType': 'T', }], }, 'spliceOutliers': { @@ -1398,7 +1398,7 @@ def test_get_individual_rna_seq_data(self): 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 3.08e-56, 'pValue': 1.08e-56, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, 'start': 132885746, 'strand': '*', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34, - 'tissueType': 'F', + 'tissueType': 'F', 'sequencingType': 'W', }, outliers_by_pos[132885746] ) diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 7382a100a2..a9bb075618 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -207,7 +207,7 @@ def test_saved_variant_data(self): 'outliers': { 'ENSG00000135953': [{ 'geneId': 'ENSG00000135953', 'zScore': 7.31, 'pValue': 0.00000000000948, 'pAdjust': 0.00000000781, - 'tissueType': 'M', 'isSignificant': True, + 'tissueType': 'M', 'sequencingType': 'T', 'isSignificant': True, }] }, 'spliceOutliers': {}, From 1094821274eb3cbaa3d3057a5b61e58d73ae7db3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 1 May 2026 12:35:23 -0400 Subject: [PATCH 062/119] fix rna page display --- .../Project/components/EditDatasetsButton.jsx | 8 +--- .../Project/components/RnaSeqResultPage.jsx | 47 +++++++++---------- ui/pages/Project/selectors.js | 22 +++++---- .../table/RnaSeqJunctionOutliersTable.jsx | 1 - ui/shared/utils/constants.js | 6 +++ 5 files changed, 43 insertions(+), 41 deletions(-) diff --git a/ui/pages/Project/components/EditDatasetsButton.jsx b/ui/pages/Project/components/EditDatasetsButton.jsx index da6b3ed99f..23b8791897 100644 --- a/ui/pages/Project/components/EditDatasetsButton.jsx +++ b/ui/pages/Project/components/EditDatasetsButton.jsx @@ -11,7 +11,7 @@ import { UPLOAD_PROJECT_IGV_FIELD } from 'shared/components/form/IGVUploadField' import FileUploadField from 'shared/components/form/XHRUploaderField' import { Select } from 'shared/components/form/Inputs' import AddWorkspaceDataForm from 'shared/components/panel/LoadWorkspaceDataForm' -import { LOAD_RNA_FIELDS, TISSUE_DISPLAY } from 'shared/utils/constants' +import { LOAD_RNA_FIELDS, TISSUE_DISPLAY, PRODUCT_DISPLAY } from 'shared/utils/constants' import { addIGVDataset, uploadRnaSeq } from '../reducers' import { getCurrentProject, getProjectGuid, getRnaSeqUploadStats } from '../selectors' @@ -76,11 +76,7 @@ const PROJECT_LOAD_RNA_FIELDS = [ name: 'sequencingType', label: 'Sequencing Type', component: Select, - options: [ - { value: 'K', name: 'Kinnex' }, - { value: 'T', name: 'TruSeq' }, - { value: 'W', name: 'Watchmaker' }, - ], + options: Object.entries(PRODUCT_DISPLAY).map(([value, name]) => ({ value, name })), validate: validators.required, }, ...LOAD_RNA_FIELDS.slice(-2), diff --git a/ui/pages/Project/components/RnaSeqResultPage.jsx b/ui/pages/Project/components/RnaSeqResultPage.jsx index 57d43210f8..0546c4916c 100644 --- a/ui/pages/Project/components/RnaSeqResultPage.jsx +++ b/ui/pages/Project/components/RnaSeqResultPage.jsx @@ -5,7 +5,7 @@ import { connect } from 'react-redux' import { Loader, Grid, Dropdown } from 'semantic-ui-react' import { getGenesById, getIndividualsByGuid, getRnaSeqSignificantJunctionData } from 'redux/selectors' -import { RNASEQ_JUNCTION_PADDING, TISSUE_DISPLAY } from 'shared/utils/constants' +import { RNASEQ_JUNCTION_PADDING } from 'shared/utils/constants' import DataLoader from 'shared/components/DataLoader' import FamilyReads from 'shared/components/panel/family/FamilyReads' import RnaSeqJunctionOutliersTable from 'shared/components/table/RnaSeqJunctionOutliersTable' @@ -46,45 +46,44 @@ class BaseRnaSeqResultPage extends React.PureComponent { } state = { - tissueType: '', + selectedDataType: null, } onTissueChange = (e, data) => { - // TODO - this.setState({ tissueType: data.value }) + this.setState({ selectedDataType: data.value }) } render() { const { familyGuid, rnaSeqData, significantJunctionOutliers, genesById, tissueOptions } = this.props - const { tissueType } = this.state - const defaultTissue = tissueOptions?.length > 0 ? tissueOptions[0].value : null - const showTissueType = tissueType === '' ? defaultTissue : tissueType - const tissueDisplay = TISSUE_DISPLAY[showTissueType] + const { selectedDataType } = this.state + const showDataType = selectedDataType || tissueOptions[0].value + const [showTissueType, showSequencingType] = showDataType.split('-') const outlierPlotConfigs = OUTLIER_VOLCANO_PLOT_CONFIGS.map((config) => { - const data = rnaSeqData[config.key] - return ({ data: data.filter(outlier => outlier.tissueType === showTissueType), ...config }) + const data = rnaSeqData[config.key].filter( + outlier => outlier.tissueType === showTissueType && outlier.sequencingType === showSequencingType, + ) + return ({ data, ...config }) }).filter(({ data }) => data.length) const tableData = significantJunctionOutliers.reduce( - (acc, outlier) => (outlier.tissueType === showTissueType ? [...acc, outlier] : acc), [], + (acc, outlier) => ( + (outlier.tissueType === showTissueType && outlier.sequencingType === showSequencingType) ? + [...acc, outlier] : acc + ), [], ) return (
- {(showTissueType || tissueOptions?.length > 1) && ( - - Tissue type:   - {tissueOptions?.length > 1 ? ( - - ) : tissueDisplay} - - )} + + {tissueOptions?.length > 1 ? ( + + ) : tissueOptions[0].text} + { outlierPlotConfigs.length > 0 && ( }> diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index e6bc8e0f4b..70a132444a 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -12,6 +12,7 @@ import { INDIVIDUAL_HAS_DATA_FIELD, MME_TAG_NAME, TISSUE_DISPLAY, + PRODUCT_DISPLAY, SIMPLIFIED_SEX_LOOKUP, } from 'shared/utils/constants' import { toCamelcase, toSnakecase, snakecaseToTitlecase } from 'shared/utils/stringUtils' @@ -852,21 +853,22 @@ export const getIndividualPhenotypeGeneScores = createSelector( export const getTissueOptionsByIndividualGuid = createSelector( getRnaSeqDataByIndividual, (rnaSeqDataByIndividualGuid) => { - // TODO const tissueTypesByIndividualGuid = Object.entries(rnaSeqDataByIndividualGuid || {}).map( ([individualGuid, rnaSeqData]) => ([ individualGuid, - [...new Set(Object.values(rnaSeqData || {}).map(Object.values).flat(2).map(({ tissueType }) => tissueType))], + [...new Set(Object.values(rnaSeqData || {}).map(Object.values).flat(2).map( + ({ tissueType, sequencingType }) => `${tissueType}-${sequencingType}`, + ))], ]), ) - return tissueTypesByIndividualGuid.reduce((acc, [individualGuid, tissueTypes]) => ( - tissueTypes.length > 0 ? { - ...acc, - [individualGuid]: tissueTypes.map(tissueType => ( - { key: tissueType, text: TISSUE_DISPLAY[tissueType] || 'Unknown Tissue', value: tissueType } - )), - } : acc - ), {}) + return tissueTypesByIndividualGuid.reduce((acc, [individualGuid, dataTypes]) => ({ + ...acc, + [individualGuid]: dataTypes.map(dataType => ({ + key: dataType, + value: dataType, + text: `Tissue type: ${TISSUE_DISPLAY[dataType.split('-')[0]] || 'Unknown Tissue'}, Sequencing Product: ${PRODUCT_DISPLAY[dataType.split('-')[1]] || 'Unknown Product'}`, + })), + }), {}) }, ) diff --git a/ui/shared/components/table/RnaSeqJunctionOutliersTable.jsx b/ui/shared/components/table/RnaSeqJunctionOutliersTable.jsx index 04e231cf45..bd0babdcd4 100644 --- a/ui/shared/components/table/RnaSeqJunctionOutliersTable.jsx +++ b/ui/shared/components/table/RnaSeqJunctionOutliersTable.jsx @@ -15,7 +15,6 @@ const RNA_SEQ_SPLICE_NUM_FIELDS = ['pValue', 'pAdjust', 'deltaIntronJaccardIndex const RNA_SEQ_SPLICE_DETAIL_FIELDS = ['type'] const openReads = (updateReads, row) => () => { - // TODO const { chrom, start, end, tissueType, familyGuid } = row updateReads(familyGuid, getLocus(chrom, start, RNASEQ_JUNCTION_PADDING, end - start), [JUNCTION_TYPE, COVERAGE_TYPE], tissueType) diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index 40a82f0f53..908acc8294 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -2119,6 +2119,12 @@ export const TISSUE_DISPLAY = { B: 'Brain', } +export const PRODUCT_DISPLAY = { + K: 'Kinnex', + T: 'TruSeq', + W: 'Watchmaker', +} + export const RNASEQ_JUNCTION_PADDING = 200 export const FAQ_PATH = '/faq' From c3b8b47526a76d5c04dabd33b71ec54e8999f6aa Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 1 May 2026 12:41:46 -0400 Subject: [PATCH 063/119] fix test --- clickhouse_search/search_tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 6801b65b4b..126235428b 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -2296,22 +2296,22 @@ def test_search_context(self): self.assertDictEqual( response_json['rnaSeqData']['I000001_na19675']['outliers']['ENSG00000268903'][0], {'geneId': 'ENSG00000268903', 'isSignificant': True, 'pAdjust': 1.39e-09, 'pValue': 5.88e-10, - 'tissueType': 'M', 'zScore': 7.08} + 'tissueType': 'M', 'sequencingType': 'T', 'zScore': 7.08} ) self.assertListEqual( sorted(response_json['rnaSeqData']['I000001_na19675']['spliceOutliers']['ENSG00000268903'], key=lambda d: d['start']), [{'chrom': '7', 'counts': 1297, 'end': 4000, 'geneId': 'ENSG00000268903', 'isSignificant': True, 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 0.0003, 'pValue': 0.0001, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, - 'start': 3000, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': -12.34}, + 'start': 3000, 'strand': '*', 'tissueType': 'F', 'sequencingType': 'W', 'type': 'psi5', 'deltaIntronJaccardIndex': -12.34}, {'chrom': '7', 'counts': 1297, 'end': 8000, 'geneId': 'ENSG00000268903', 'isSignificant': True, 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 0.003, 'pValue': 0.001, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, - 'start': 7000, 'strand': '*', 'tissueType': 'M', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}, + 'start': 7000, 'strand': '*', 'tissueType': 'M', 'sequencingType': 'T', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}, {'chrom': '7', 'counts': 1297, 'end': 132886973, 'geneId': 'ENSG00000268903', 'isSignificant': True, 'meanCounts': 0.85, 'meanTotalCounts': 0.85, 'pAdjust': 3.08e-56, 'pValue': 1.08e-56, 'rareDiseaseSamplesTotal': 20, 'rareDiseaseSamplesWithThisJunction': 1, 'totalCounts': 1297, - 'start': 132885746, 'strand': '*', 'tissueType': 'F', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}] + 'start': 132885746, 'strand': '*', 'tissueType': 'F', 'sequencingType': 'W', 'type': 'psi5', 'deltaIntronJaccardIndex': 12.34}] ) self.assertDictEqual(response_json['savedVariantsByGuid'], {'SV0000001_2103343353_r0390_100': { **{k: mock.ANY for k in [ From 6f38d14e2760fd94f78e19eb6a2b883a8524d214 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 1 May 2026 13:02:39 -0400 Subject: [PATCH 064/119] update tpm display --- .../components/panel/variants/RnaSeqTpm.jsx | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/ui/shared/components/panel/variants/RnaSeqTpm.jsx b/ui/shared/components/panel/variants/RnaSeqTpm.jsx index e68abf3ff2..6185d61d3a 100644 --- a/ui/shared/components/panel/variants/RnaSeqTpm.jsx +++ b/ui/shared/components/panel/variants/RnaSeqTpm.jsx @@ -4,17 +4,17 @@ import { extent, max, median, min, quantile } from 'd3-array' import { scaleBand, scaleLinear } from 'd3-scale' import { HttpRequestHelper } from 'shared/utils/httpRequestHelper' -import { TISSUE_DISPLAY } from 'shared/utils/constants' +import { TISSUE_DISPLAY, PRODUCT_DISPLAY } from 'shared/utils/constants' import { initializeD3, Tooltip } from 'shared/components/graph/d3Utils' import GtexLauncher from 'shared/components/graph/GtexLauncher' const BOX_WIDTH = 100 -const PLOT_WIDTH = 550 +const PLOT_WIDTH = 500 const PLOT_HEIGHT = 350 const MARGINS = { left: 40, top: 0, - bottom: 100, + bottom: 130, } // Code adapted from https://github.com/broadinstitute/gtex-viz/blob/8d65862fbe7e5ab9b4d5be419568754e0d17bb07/src/modules/Boxplot.js @@ -56,7 +56,7 @@ const renderBoxplot = (allData, containerElement) => { const svg = initializeD3( containerElement, { width: PLOT_WIDTH, height: PLOT_HEIGHT }, - { ...MARGINS, right: bandwidth / 2 + 10 }, + { ...MARGINS, right: bandwidth / 2 + 60 }, scales, { x: { offset: bandwidth / 2 }, y: { text: 'TPM' } }, ) @@ -189,18 +189,16 @@ const renderGtex = (gtexExpressionData, familyExpressionData, containerElement) const gtexByTissue = ((gtexExpressionData || {}).data || []).reduce((acc, { data, tissueSiteDetailId }) => ({ ...acc, [GTEX_TISSUE_LOOKUP[tissueSiteDetailId]]: data, }), {}) - // TODO - const boxplotData = Object.entries(familyExpressionData).reduce(( - acc, [tissue, { rdgData, myData, individualData }], - ) => ([ - ...acc, - ...[ - { data: gtexByTissue[tissue], label: 'GTEx' }, - { data: rdgData, label: 'Broad RDG' }, - { data: myData, label: 'My Data' }, - ...Object.entries(individualData).map(([individual, tpm]) => ({ data: [tpm], label: individual, medianColor: '#000080' })), - ].filter(({ data }) => data).map(({ label, ...d }) => ({ label: `${label} - ${TISSUE_DISPLAY[tissue]}`, ...d })), - ]), []) + const boxplotData = Object.entries(familyExpressionData).reduce((acc, [tissue, tissueData]) => ([ + ...acc, ...[ + { data: gtexByTissue[tissue], label: `GTEx - ${TISSUE_DISPLAY[tissue]}` }, + ...Object.entries(tissueData).reduce((tAcc, [sequencingProduct, { rdgData, myData, individualData }]) => ([ + ...tAcc, + { data: rdgData, label: 'Broad RDG' }, + { data: myData, label: 'My Data' }, + ...Object.entries(individualData).map(([individual, tpm]) => ({ data: [tpm], label: individual, medianColor: '#000080' })), + ].filter(({ data }) => data).map(({ label, ...d }) => ({ label: `${label} - ${TISSUE_DISPLAY[tissue]} - ${PRODUCT_DISPLAY[sequencingProduct]}`, ...d }))), []), + ]]), []) renderBoxplot(boxplotData, containerElement) } From eb80aa2e56a6b498a6377e5effae080b2b33cf76 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 1 May 2026 13:45:25 -0400 Subject: [PATCH 065/119] fix airtable sequencing mapping --- seqr/views/apis/data_manager_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 7803d8d2c5..2aa72bd842 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -81,7 +81,7 @@ def update_rna_seq(request): elif not pdos[0][SEQUENCING_FIELD]: error_messages.append('no sequencing product specified') else: - sequencing_type = next((st for st, label in RnaSample.SEQUENCING_CHOICES if pdos[0][SEQUENCING_FIELD].startswith(st)), None) + sequencing_type = next((st for st, label in RnaSample.SEQUENCING_CHOICES if pdos[0][SEQUENCING_FIELD].startswith(label)), None) if not sequencing_type: error_messages.append('invalid sequencing product specified') From c7b2fed34a489004cd1c3b13fbe148c64f9d2aa1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 1 May 2026 14:32:30 -0400 Subject: [PATCH 066/119] return saved search name --- seqr/views/utils/orm_to_json_utils.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 01513afab9..62d48f16e5 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -4,7 +4,7 @@ from collections import defaultdict from django.contrib.postgres.aggregates import ArrayAgg -from django.db.models import prefetch_related_objects, Count, Value, F, Q, CharField, Case, When +from django.db.models import prefetch_related_objects, Count, Value, F, Q, CharField, Case, When, OuterRef, Subquery from django.db.models.functions import Concat, Coalesce, NullIf, Lower, Trim, JSONObject from django.contrib.auth.models import User from guardian.shortcuts import get_users_with_perms, get_groups_with_perms @@ -13,7 +13,7 @@ from panelapp.models import PaLocusList from reference_data.models import HumanPhenotypeOntology from seqr.models import GeneNote, VariantNote, VariantTag, VariantFunctionalData, SavedVariant, Family, CAN_VIEW, CAN_EDIT, \ - get_audit_field_names, RnaSeqOutlier, RnaSeqSpliceOutlier + get_audit_field_names, RnaSeqOutlier, RnaSeqSpliceOutlier, VariantSearchResults from seqr.utils.xpos_utils import get_chrom_pos from seqr.views.utils.json_utils import _to_camel_case from seqr.views.utils.permissions_utils import has_project_permissions, \ @@ -452,14 +452,18 @@ def get_json_for_saved_variants_child_entities(tag_cls, saved_variant_id_map, ta guid_key = 'tagGuid' nested_fields = None + additional_model_fields = ['id'] if tag_cls == VariantTag: nested_fields = [ {'fields': ('variant_tag_type', field), 'key': field} for field in ['name', 'category', 'color']] + related_searches = VariantSearchResults.objects.filter(search_hash=OuterRef('search_hash')).values('variant_search__name') + tag_models = tag_models.annotate(search_name=Subquery(related_searches[:1])) + additional_model_fields.append('search_name') elif tag_cls == VariantNote: guid_key = 'noteGuid' tags = get_json_for_queryset( - tag_models, guid_key=guid_key, nested_fields=nested_fields, additional_model_fields=['id']) + tag_models, guid_key=guid_key, nested_fields=nested_fields, additional_model_fields=additional_model_fields) if tag_cls == VariantFunctionalData: _format_functional_tags(tags) elif tag_cls == VariantTag: From d04fed64c49732358a957633949bf9fe0dc314d4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 1 May 2026 16:36:40 -0400 Subject: [PATCH 067/119] show search name in ui --- .../components/panel/variants/FamilyVariantTags.jsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ui/shared/components/panel/variants/FamilyVariantTags.jsx b/ui/shared/components/panel/variants/FamilyVariantTags.jsx index 341421613f..16b888e6ec 100644 --- a/ui/shared/components/panel/variants/FamilyVariantTags.jsx +++ b/ui/shared/components/panel/variants/FamilyVariantTags.jsx @@ -108,7 +108,14 @@ export const taggedByPopup = (tag, title) => (trigger, hideMetadata) => ( ) : {tag.metadata}}
)} - {tag.searchHash &&
Re-run search
} + {tag.searchHash && ( +
+ + Re-run   + {tag.searchName ? `search: "${tag.searchName}"` : 'custom search'} + +
+ )} } /> From 69060098e8eb50bb63abae796e07df9e15389df7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 1 May 2026 16:54:27 -0400 Subject: [PATCH 068/119] update tests --- clickhouse_search/search_tests.py | 1 + seqr/fixtures/1kg_project.json | 18 ++++++++++++++++++ seqr/views/apis/saved_variant_api_tests.py | 13 +++++++++++++ seqr/views/utils/test_utils.py | 2 +- 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 6801b65b4b..76a9f20423 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -2404,6 +2404,7 @@ def test_search_context(self): 'category': 'CMG Discovery Tags', 'color': '#03441E', 'searchHash': None, + 'searchName': None, 'metadata': None, 'lastModifiedDate': '2018-05-29T16:32:51.449Z', 'createdBy': None, diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index 68a415bcdc..ff0c410705 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -1940,6 +1940,24 @@ "search_hash": "125897d6" } }, +{ + "model": "seqr.variantsearchresults", + "pk": 123, + "fields": { + "guid": "VSR000123_125897d6", + "variant_search_id": 1234, + "search_hash": "125897d6" + } +}, + { + "model": "seqr.variantsearch", + "pk": 1234, + "fields": { + "guid": "VS0001234_incomplete_penetranc", + "name": "Incomplete Penetrance De-novo", + "search": {} + } +}, { "model": "seqr.varianttag", "pk": 1726945, diff --git a/seqr/views/apis/saved_variant_api_tests.py b/seqr/views/apis/saved_variant_api_tests.py index 7382a100a2..c16c574dd2 100644 --- a/seqr/views/apis/saved_variant_api_tests.py +++ b/seqr/views/apis/saved_variant_api_tests.py @@ -184,6 +184,18 @@ def test_saved_variant_data(self): tag = response_json['variantTagsByGuid']['VT1708633_2103343353_r0390_100'] self.assertSetEqual(set(tag.keys()), TAG_FIELDS) + self.assertDictEqual(tag, { + 'category': 'Collaboration', + 'color': '#668FE3', + 'createdBy': None, + 'lastModifiedDate': '2018-02-23T17:32:30.373Z', + 'metadata': None, + 'name': 'Review', + 'searchHash': '125897d6', + 'searchName': 'Incomplete Penetrance De-novo', + 'tagGuid': 'VT1708633_2103343353_r0390_100', + 'variantGuids': [VARIANT_GUID], + }) submissions = response_json['mmeSubmissionsByGuid'] self.assertSetEqual(set(submissions.keys()), {'MS000001_na19675'}) @@ -341,6 +353,7 @@ def test_saved_variant_data(self): 'category': 'CMG Discovery Tags', 'color': '#03441E', 'searchHash': None, + 'searchName': None, 'metadata': None, 'lastModifiedDate': '2018-05-29T16:32:51.449Z', 'createdBy': None, diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index 02335af03e..8b628a1210 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -721,7 +721,7 @@ def _get_list_param(call, param): SAVED_VARIANT_DETAIL_FIELDS.update(SAVED_VARIANT_FIELDS) TAG_FIELDS = { - 'tagGuid', 'name', 'category', 'color', 'searchHash', 'metadata', 'lastModifiedDate', 'createdBy', 'variantGuids', + 'tagGuid', 'name', 'category', 'color', 'searchHash', 'searchName', 'metadata', 'lastModifiedDate', 'createdBy', 'variantGuids', } VARIANT_NOTE_FIELDS = {'noteGuid', 'note', 'report', 'lastModifiedDate', 'createdBy', 'variantGuids'} From d320dc26f1a15f274522b9d38b6fee44e76e461b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 1 May 2026 17:22:58 -0400 Subject: [PATCH 069/119] fix tests --- seqr/views/apis/variant_search_api_tests.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index 631db9c59a..3757d58ddc 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -23,6 +23,7 @@ EXPECTED_SEARCH_CONTEXT_RESPONSE = { 'savedSearchesByGuid': { 'VS0079516_': mock.ANY, 'VS0079525_': mock.ANY, 'VS0079517_': mock.ANY, 'VS0145435_': mock.ANY, + 'VS0001234_incomplete_penetranc': mock.ANY, }, 'projectsByGuid': {PROJECT_GUID: mock.ANY}, 'familiesByGuid': mock.ANY, @@ -211,7 +212,7 @@ def test_search_context(self): expected_response['projectsByGuid']['R0003_test'] = mock.ANY self.assertSetEqual(set(response_json), set(expected_response)) self.assertDictEqual(response_json, expected_response) - self.assertEqual(len(response_json['savedSearchesByGuid']), 4) + self.assertEqual(len(response_json['savedSearchesByGuid']), 5) self.assertSetEqual(set(response_json['projectsByGuid'][PROJECT_GUID].keys()), PROJECT_CONTEXT_FIELDS) self.assertSetEqual(set(response_json['projectsByGuid'][PROJECT_GUID]['datasetTypes']), {'SNV_INDEL', 'SV', 'MITO'}) self.assertSetEqual(set(response_json['projectsByGuid']['R0003_test']['datasetTypes']), {'SNV_INDEL'}) @@ -344,7 +345,7 @@ def test_saved_search(self): response = self.client.get(get_saved_search_url) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.json()['savedSearchesByGuid']), 4) + self.assertEqual(len(response.json()['savedSearchesByGuid']), 5) create_saved_search_url = reverse(create_saved_search_handler) @@ -379,7 +380,7 @@ def test_saved_search(self): response = self.client.get(get_saved_search_url) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.json()['savedSearchesByGuid']), 5) + self.assertEqual(len(response.json()['savedSearchesByGuid']), 6) # Test cannot save different searches with the same name body['filters'] = {'test': 'filter'} @@ -409,7 +410,7 @@ def test_saved_search(self): response = self.client.get(get_saved_search_url) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.json()['savedSearchesByGuid']), 4) + self.assertEqual(len(response.json()['savedSearchesByGuid']), 5) global_saved_search_guid = next(iter(response.json()['savedSearchesByGuid'])) From 244cc5586b39cd6e21447dfc8a458b81c55b321f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 6 May 2026 16:57:24 -0400 Subject: [PATCH 070/119] docstring cleanup --- clickhouse_search/models/__init__.py | 4 +- matchmaker/views/external_api.py | 23 ---- matchmaker/views/matchmaker_api.py | 40 ------- panelapp/models.py | 6 - reference_data/models.py | 22 +--- reference_data/utils/download_utils.py | 7 -- .../commands/reset_cached_search_results.py | 1 - seqr/models.py | 37 +------ seqr/urls.py | 5 - seqr/utils/file_utils.py | 1 - seqr/utils/xpos_utils.py | 7 -- seqr/views/apis/anvil_workspace_api.py | 27 ----- seqr/views/apis/auth_api.py | 12 -- seqr/views/apis/awesomebar_api.py | 14 +-- seqr/views/apis/case_review_api.py | 3 - seqr/views/apis/dashboard_api.py | 23 ---- seqr/views/apis/family_api.py | 40 +------ seqr/views/apis/feature_updates_api.py | 4 +- seqr/views/apis/project_api.py | 58 ---------- seqr/views/apis/project_categories_api.py | 34 ------ seqr/views/apis/variant_search_api.py | 6 - seqr/views/utils/export_utils.py | 2 + seqr/views/utils/individual_utils.py | 25 ----- seqr/views/utils/json_utils.py | 15 +-- seqr/views/utils/orm_to_json_utils.py | 104 ------------------ seqr/views/utils/pedigree_info_utils.py | 76 ------------- seqr/views/utils/permissions_utils.py | 8 -- seqr/views/utils/terra_api_utils.py | 27 ----- seqr/views/utils/test_utils.py | 11 +- wsgi.py | 15 --- 30 files changed, 12 insertions(+), 645 deletions(-) diff --git a/clickhouse_search/models/__init__.py b/clickhouse_search/models/__init__.py index a34e5c1e9f..a1c506ef4a 100644 --- a/clickhouse_search/models/__init__.py +++ b/clickhouse_search/models/__init__.py @@ -13,9 +13,7 @@ state.DEFAULT_NAMES = options.DEFAULT_NAMES class ClickHouseRouter: - """ - Adapted from https://github.com/jayvynl/django-clickhouse-backend/blob/v1.3.2/README.md#configuration - """ + # Adapted from https://github.com/jayvynl/django-clickhouse-backend/blob/v1.3.2/README.md#configuration def __init__(self): self.route_model_names = set() diff --git a/matchmaker/views/external_api.py b/matchmaker/views/external_api.py index f6637fe456..e57e3e25e8 100644 --- a/matchmaker/views/external_api.py +++ b/matchmaker/views/external_api.py @@ -45,26 +45,12 @@ def _wrapped_view(request, *args, **kwargs): @authenticate_mme_request def mme_metrics_proxy(request, originating_node_name): - """ - -Proxies public metrics endpoint - Returns: - Metric JSON from matchbox - """ logger.info('Received MME metrics request from {}'.format(originating_node_name)) return create_json_response({'metrics': get_mme_metrics()}) @authenticate_mme_request def mme_match_proxy(request, originating_node_name): - """ - -Looks for matches for the given individual ONLY in the local MME DB. - -Expects a single patient (as per MME spec) in the POST - - Args: - None, all data in POST under key "patient_data" - Returns: - Status code and results (as per MME spec), returns raw results from MME Server - """ logger.info('Received MME match request from {}'.format(originating_node_name)) try: @@ -88,15 +74,6 @@ def mme_match_proxy(request, originating_node_name): def _safe_generate_notification_for_incoming_match(results, incoming_query, incoming_request_node, incoming_patient): - """ - Generate a SLACK notifcation to say that a VALID match request came in and the following - results were sent back. If Slack is not supported, a message is not sent, but details persisted. - - Args: - response_from_matchbox (python requests object): contains the response from matchbox - incoming_request (Django request object): The request that came into the view - incoming_patient (JSON): The query patient JSON structure from outside MME node that was matched with - """ incoming_patient_id = incoming_patient['patient']['id'] logger.info('{} MME matches found for patient {} from {}'.format( diff --git a/matchmaker/views/matchmaker_api.py b/matchmaker/views/matchmaker_api.py index 7c7abadc07..c261a5ee7f 100644 --- a/matchmaker/views/matchmaker_api.py +++ b/matchmaker/views/matchmaker_api.py @@ -35,14 +35,6 @@ @login_and_policies_required def get_individual_mme_matches(request, submission_guid): - """ - Looks for matches for the given submission. Expects a single patient (MME spec) in the POST - data field under key "patient_data" - Args: - project_id,indiv_id and POST all data in POST under key "patient_data" - Returns: - Status code and results - """ submission = MatchmakerSubmission.objects.get(guid=submission_guid) project = check_mme_permissions(submission, request.user) @@ -257,9 +249,6 @@ def _report_external_mme_error(node_name, error, detail, user, raise_exception=F @login_and_policies_required def update_mme_submission(request, submission_guid=None): - """ - Create or update the submission for the given individual. - """ submission_json = json.loads(request.body) phenotypes = submission_json.pop('phenotypes', []) gene_variants = submission_json.pop('geneVariants', []) @@ -332,9 +321,6 @@ def _get_submission_detail_response(submission, phenotypes): @login_and_policies_required def delete_mme_submission(request, submission_guid): - """ - Create or update the submission for the given individual. - """ submission = MatchmakerSubmission.objects.get(guid=submission_guid) check_mme_permissions(submission, request.user) @@ -358,14 +344,6 @@ def delete_mme_submission(request, submission_guid): @login_and_policies_required def update_mme_result_status(request, matchmaker_result_guid): - """ - Looks for matches for the given individual. Expects a single patient (MME spec) in the POST - data field under key "patient_data" - Args: - project_id,indiv_id and POST all data in POST under key "patient_data" - Returns: - Status code and results - """ result = MatchmakerResult.objects.get(guid=matchmaker_result_guid) check_mme_permissions(result.submission, request.user) @@ -380,13 +358,6 @@ def update_mme_result_status(request, matchmaker_result_guid): @login_and_policies_required def send_mme_contact_email(request, matchmaker_result_guid): - """ - Sends the given email and updates the contacted status for the match - Args: - matchmaker_result_guid - Returns: - Status code and results - """ result = MatchmakerResult.objects.get(guid=matchmaker_result_guid) check_mme_permissions(result.submission, request.user) @@ -441,14 +412,6 @@ def update_mme_project_contact(request, project_guid): @analyst_required def update_mme_contact_note(request, institution): - """ - Looks for matches for the given individual. Expects a single patient (MME spec) in the POST - data field under key "patient_data" - Args: - project_id,indiv_id and POST all data in POST under key "patient_data" - Returns: - Status code and results - """ institution = institution.strip().lower() request_json = json.loads(request.body) note, _ = get_or_create_model_from_json( @@ -509,9 +472,6 @@ def _parse_mme_result(result, hpo_terms_by_id, gene_symbols_to_ids, submission_g def _generate_notification_for_seqr_match(submission, results): - """ - Generate a notifcation to say that a match happened initiated from a seqr user. - """ matches = [] hpo_terms_by_id, genes_by_id, _ = get_mme_genes_phenotypes_for_results(results) for result in results: diff --git a/panelapp/models.py b/panelapp/models.py index 9e58d53c5c..7a10f17dd3 100644 --- a/panelapp/models.py +++ b/panelapp/models.py @@ -14,8 +14,6 @@ class PaLocusList(models.Model): - """PanelApp extension of seqr.models.LocusList.""" - seqr_locus_list = models.OneToOneField( SeqrLocusList, on_delete=models.CASCADE, @@ -38,8 +36,6 @@ class Meta: class PaLocusListGene(models.Model): - """PanelApp extension of seqr.models.LocusListGene.""" - CONFIDENCE_LEVEL_CHOICES = [ ('1', 'RED'), ('2', 'AMBER'), @@ -60,8 +56,6 @@ class PaLocusListGene(models.Model): mode_of_inheritance = models.TextField(null=True, blank=True) class Meta: - """Fields included in JSON in API calls.""" - json_fields = ['confidence_level', 'mode_of_inheritance'] diff --git a/reference_data/models.py b/reference_data/models.py index 91eed9fdf0..8aad05578c 100644 --- a/reference_data/models.py +++ b/reference_data/models.py @@ -31,32 +31,22 @@ class ReferenceDataRouter(object): - """ - A router to control all database operations on reference data models - """ + """A router to control all database operations on reference data models.""" + @classmethod def db_for_read(cls, model, **hints): - """ - Attempts to read reference_data models go to reference_data_db. - """ if model._meta.app_label == 'reference_data': return 'reference_data' return None @classmethod def db_for_write(cls, model, **hints): - """ - Attempts to write reference_data models go to reference_data_db. - """ if model._meta.app_label == 'reference_data': return 'reference_data' return None @classmethod def allow_relation(cls, obj1, obj2, **hints): - """ - Allow relations if a model in the reference_data app is involved. - """ if obj1._meta.app_label == 'reference_data' or \ obj2._meta.app_label == 'reference_data': return True @@ -64,10 +54,6 @@ def allow_relation(cls, obj1, obj2, **hints): @classmethod def allow_migrate(cls, db, app_label, model_name=None, **hints): - """ - Make sure the reference_data app only appears in the 'reference_data_db' - database. - """ if app_label == 'reference_data': return db == 'reference_data' elif db == 'reference_data': @@ -241,10 +227,6 @@ def _get_category_id(parent_id_map, hpo_id): return hpo_id class GeneInfo(LoadableModel): - """Human gene models from https://www.gencodegenes.org/releases/ - http://www.gencodegenes.org/gencodeformat.html - """ - ALL_GENCODE_VERSIONS = ['39', '31', '29', '28', '27', '19'] CURRENT_VERSION = ALL_GENCODE_VERSIONS[0] diff --git a/reference_data/utils/download_utils.py b/reference_data/utils/download_utils.py index 97c32148d0..c6f29b14f1 100644 --- a/reference_data/utils/download_utils.py +++ b/reference_data/utils/download_utils.py @@ -8,13 +8,6 @@ def download_file(url): - """Download the given file and returns its local path. - Args: - url (string): HTTP or FTP url - Returns: - string: local file path - """ - to_dir = tempfile.gettempdir() if not (url and url.startswith(("http://", "https://"))): raise ValueError("Invalid url: {}".format(url)) diff --git a/seqr/management/commands/reset_cached_search_results.py b/seqr/management/commands/reset_cached_search_results.py index 5adcbb33c4..8b75e72366 100644 --- a/seqr/management/commands/reset_cached_search_results.py +++ b/seqr/management/commands/reset_cached_search_results.py @@ -14,7 +14,6 @@ def add_arguments(self, parser): parser.add_argument('--reset-index-metadata', action='store_true', help='reset the index metadata') def handle(self, *args, **options): - """transfer project""" project_name = options['project'] project = Project.objects.get(name=project_name) if project_name else None reset_cached_search_results(project=project, reset_index_metadata=options.get('reset_index_metadata')) diff --git a/seqr/models.py b/seqr/models.py index b2f53553cf..149dd30655 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -85,16 +85,12 @@ def __unicode__(self): return self.guid def __str__(self): - """Magix function for str() and %s.""" return self.__unicode__() def json(self): - """Utility method that returns a json {field-name: value-as-string} mapping for all fields.""" return {k: v for k, v in self.__dict__.items() if not k.startswith('_')} def save(self, *args, **kwargs): - """Create a GUID at object creation time.""" - being_created = not self.pk current_time = timezone.now() @@ -113,11 +109,11 @@ def save(self, *args, **kwargs): self.created_date = kwargs.pop('created_date', current_time) super(ModelWithGUID, self).save(*args, **kwargs) + # Create a GUID at object creation time self.guid = self._compute_guid() super(ModelWithGUID, self).save() def delete_model(self, user, user_can_delete=False): - """Helper delete method that logs the deletion""" if not (user_can_delete or self.created_by == user): raise PermissionDenied('User does not have permission to delete this {}'.format(type(self).__name__)) self.delete() @@ -125,7 +121,6 @@ def delete_model(self, user, user_can_delete=False): @classmethod def bulk_create(cls, user, new_models, **kwargs): - """Helper bulk create method that logs the creation""" for model in new_models: model.created_by = user model.created_date = timezone.now() @@ -136,7 +131,6 @@ def bulk_create(cls, user, new_models, **kwargs): @classmethod def bulk_update(cls, user, update_json, queryset=None, **filter_kwargs): - """Helper bulk update method that logs the update""" if queryset is None: queryset = cls.objects.filter(**filter_kwargs).exclude(**update_json) @@ -149,13 +143,11 @@ def bulk_update(cls, user, update_json, queryset=None, **filter_kwargs): @classmethod def bulk_update_models(cls, user, models, fields): - """Helper bulk update method that logs the update and allows different update data for each model""" log_model_bulk_update(logger, models, user, 'update', update_fields=fields) cls.objects.bulk_update(models, fields) @classmethod def bulk_delete(cls, user, queryset=None, **filter_kwargs): - """Helper bulk delete method that logs the deletion""" if queryset is None: queryset = cls.objects.filter(**filter_kwargs) log_model_bulk_update(logger, queryset, user, 'delete') @@ -219,10 +211,6 @@ def __unicode__(self): GUID_PRECISION = 4 def save(self, *args, **kwargs): - """Override the save method and create user permissions groups + add the created_by user. - - This could be done with signals, but seems cleaner to do it this way. - """ being_created = not self.pk anvil_disabled = not anvil_enabled() @@ -249,8 +237,6 @@ def save(self, *args, **kwargs): user.groups.add(*[getattr(self, group) for group in groups]) def delete(self, *args, **kwargs): - """Override the delete method to also delete the project-specific user groups""" - super(Project, self).delete(*args, **kwargs) if self.can_edit_group: @@ -778,9 +764,6 @@ class Meta: class IgvSample(ModelWithGUID): - """This model represents a single data type that can be displayed in IGV (eg. Read Alignments) that's generated from - a single biological sample (eg. WES, WGS, RNA, Array). - """ SAMPLE_TYPE_ALIGNMENT = 'alignment' SAMPLE_TYPE_COVERAGE = 'wig' SAMPLE_TYPE_JUNCTION = 'spliceJunctions' @@ -857,20 +840,6 @@ class Meta: class VariantTagType(ModelWithGUID): - """ - Previous color choices: - '#1f78b4', - '#a6cee3', - '#b2df8a', - '#33a02c', - '#fdbf6f', - '#ff7f00', - '#ff0000', - '#cab2d6', - '#6a3d9a', - '#8F754F', - '#383838', - """ project = models.ForeignKey('Project', null=True, blank=True, on_delete=models.CASCADE) name = models.TextField() @@ -1045,8 +1014,6 @@ class Meta: class LocusList(ModelWithGUID): - """List of gene ids or regions""" - name = models.TextField(db_index=True) description = models.TextField(null=True, blank=True) @@ -1176,7 +1143,6 @@ def log_model_no_guid_bulk_update(cls, models, user, update_type): @classmethod def bulk_create(cls, user, new_models, **kwargs): - """Helper bulk create method that logs the creation""" for model in new_models: model.created_by = user models = cls.objects.bulk_create(new_models, **kwargs) @@ -1185,7 +1151,6 @@ def bulk_create(cls, user, new_models, **kwargs): @classmethod def bulk_delete(cls, user, queryset=None, **filter_kwargs): - """Helper bulk delete method that logs the deletion""" if queryset is None: queryset = cls.objects.filter(**filter_kwargs) cls.log_model_no_guid_bulk_update(queryset, user, 'delete') diff --git a/seqr/urls.py b/seqr/urls.py index 4b2e11a7d9..5c6bf5cef7 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -1,8 +1,3 @@ -"""seqr URL Configuration - -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/1.9/topics/http/urls/ -""" from seqr.views.react_app import main_app, no_login_main_app from seqr.views.status import status_view from settings import ENABLE_DJANGO_DEBUG_TOOLBAR, MEDIA_ROOT, API_LOGIN_REQUIRED_URL, LOGIN_URL, DEBUG, \ diff --git a/seqr/utils/file_utils.py b/seqr/utils/file_utils.py index ce342250a2..cab280ed6f 100644 --- a/seqr/utils/file_utils.py +++ b/seqr/utils/file_utils.py @@ -83,7 +83,6 @@ def file_iter(file_path, byte_range=None, raw_content=False, user=None, **kwargs def _google_bucket_file_iter(gs_path, byte_range=None, raw_content=False, user=None, **kwargs): - """Iterate over lines in the given file""" range_arg = ' -r {}-{}'.format(byte_range[0], byte_range[1]) if byte_range else '' process = _run_gsutil_command( 'cat{}'.format(range_arg), gs_path, gunzip=gs_path.endswith("gz") and not raw_content, user=user, **kwargs) diff --git a/seqr/utils/xpos_utils.py b/seqr/utils/xpos_utils.py index 6d06eac680..d0619b3ccd 100644 --- a/seqr/utils/xpos_utils.py +++ b/seqr/utils/xpos_utils.py @@ -56,12 +56,6 @@ def format_chrom(chrom): def get_xpos(chrom, pos): - """Compute single number representing this chromosome and position. - - Args: - chrom (string): examples '1', 'Y', 'M' - pos (integer): genomic position on chromosome - """ chrom = format_chrom(chrom) if pos < MIN_POS or pos > MAX_POS: @@ -71,7 +65,6 @@ def get_xpos(chrom, pos): def get_chrom_pos(xpos): - """Converts xpos position to a (chr, pos) tuple""" chrom_idx = int(xpos/1e9) - 1 if chrom_idx < 0 or chrom_idx >= len(CHROMOSOMES): raise ValueError("Invalid xpos: %s" % (xpos,)) diff --git a/seqr/views/apis/anvil_workspace_api.py b/seqr/views/apis/anvil_workspace_api.py index 4b08c4e2af..f673dc0431 100644 --- a/seqr/views/apis/anvil_workspace_api.py +++ b/seqr/views/apis/anvil_workspace_api.py @@ -1,4 +1,3 @@ -"""APIs for management of projects related to AnVIL workspaces.""" import json import time from datetime import datetime @@ -58,15 +57,6 @@ def _wrapped_view(request, namespace, name, *args, **kwargs): @anvil_auth_and_policies_required(policy_url=POLICY_REQUIRED_URL) def anvil_workspace_page(request, namespace, name): - """ - This view will be requested from AnVIL, it validates the workspace and project before loading data. - - :param request: Django request object. - :param namespace: The namespace (or the billing account) of the workspace. - :param name: The name of the workspace. It also be used as the project name. - :return Redirect to a page depending on if the workspace permissions or project exists. - - """ project = Project.objects.filter(workspace_namespace=namespace, workspace_name=name) if project: return redirect('/project/{}/project_page'.format(project.first().guid)) @@ -159,15 +149,6 @@ def validate_anvil_vcf(request, namespace, name, workspace_meta): @anvil_workspace_access_required def create_project_from_workspace(request, namespace, name): - """ - Create a project when a cooperator requests to load data from an AnVIL workspace. - - :param request: Django request object - :param namespace: The namespace (or the billing account) of the workspace - :param name: The name of the workspace. It also be used as the project name - :return the projectsByGuid with the new project json - - """ projects = Project.objects.filter(workspace_namespace=namespace, workspace_name=name) if projects: error = 'Project "{}" for workspace "{}/{}" exists.'.format(projects.first().name, namespace, name) @@ -204,14 +185,6 @@ def create_project_from_workspace(request, namespace, name): @anvil_auth_and_policies_required def add_workspace_data(request, project_guid): - """ - Add data from an AnVIL workspace. - - :param request: Django request object - :param project_guid: Django request object - :return a data json with fields of individualGuid, familyGuid and optional familyNotesByGuid if no exceptions - - """ project = Project.objects.get(guid=project_guid) check_workspace_perm(request.user, CAN_EDIT, project.workspace_namespace, project.workspace_name, can_share=True) diff --git a/seqr/views/apis/auth_api.py b/seqr/views/apis/auth_api.py index 942ec3dbf3..74211df0d6 100644 --- a/seqr/views/apis/auth_api.py +++ b/seqr/views/apis/auth_api.py @@ -1,6 +1,3 @@ -""" -Utility functions related to authentication. -""" from django.contrib.auth import login, authenticate, logout from django.contrib.auth.decorators import login_required from django.contrib.auth.models import User @@ -59,23 +56,14 @@ def logout_view(request): def app_login_required_error(request, exception=None): - """Redirect to login for unhandled 401 error on non-API request""" return redirect(f'{LOGIN_URL}?next={request.path}') def login_required_error(request): - """Returns an HttpResponse with a 401 UNAUTHORIZED error message. - - This is used to redirect AJAX HTTP handlers to the login page. - """ return _unauthorized_error(LOGIN_URL) def policies_required_error(request): - """Returns an HttpResponse with a 401 UNAUTHORIZED error message. - - This is used to redirect AJAX HTTP handlers to the accept policies page. - """ return _unauthorized_error(POLICY_REQUIRED_URL) diff --git a/seqr/views/apis/awesomebar_api.py b/seqr/views/apis/awesomebar_api.py index 3685f69935..efbb17f05a 100644 --- a/seqr/views/apis/awesomebar_api.py +++ b/seqr/views/apis/awesomebar_api.py @@ -1,4 +1,4 @@ -"""API that generates auto-complete suggestions for the search bar in the header of seqr pages""" +"""API that generates auto-complete suggestions for the search bar in the header of seqr pages.""" from django.db.models import F, Q, Case, When, Value, ExpressionWrapper, BooleanField, CharField from django.db.models.functions import Cast, Coalesce, Concat, Left, Length, NullIf, Replace from django.views.decorators.http import require_GET @@ -119,14 +119,6 @@ def _get_matching_project_groups(query, project_guids): def _get_matching_genes(query, *args): - """Returns genes that match the given query string, and that the user can view. - - Args: - user: Django user - query: String typed into the awesomebar - Returns: - Sorted list of matches where each match is a dictionary of strings - """ result = [] for g in get_queried_genes(query, MAX_RESULTS_PER_CATEGORY): if query.lower() in g['gene_id'].lower(): @@ -147,7 +139,6 @@ def _get_matching_genes(query, *args): def _get_matching_omim(query, *args): - """Returns OMIM records that match the given query string""" records = Omim.objects.filter( Q(phenotype_mim_number__icontains=query) | Q(phenotype_description__icontains=query) ).filter(phenotype_mim_number__isnull=False).annotate( @@ -167,7 +158,6 @@ def _get_matching_omim(query, *args): def _get_matching_hpo_terms(query, *args): - """Returns OMIM records that match the given query string""" records = HumanPhenotypeOntology.objects.filter( Q(hpo_id__icontains=query) | Q(name__icontains=query) ).annotate( @@ -206,8 +196,6 @@ def _get_matching_hpo_terms(query, *args): @login_and_policies_required @require_GET def awesomebar_autocomplete_handler(request): - """Accepts HTTP GET request with q=.. url arg, and returns suggestions""" - query = request.GET.get('q').strip() if not query: return create_json_response({'matches': {}}) diff --git a/seqr/views/apis/case_review_api.py b/seqr/views/apis/case_review_api.py index fd1770f069..2f0687ca08 100644 --- a/seqr/views/apis/case_review_api.py +++ b/seqr/views/apis/case_review_api.py @@ -1,6 +1,3 @@ -""" -APIs used by the case review page -""" import json from django.core.exceptions import PermissionDenied diff --git a/seqr/views/apis/dashboard_api.py b/seqr/views/apis/dashboard_api.py index d11a4d9bc1..135f064fd9 100644 --- a/seqr/views/apis/dashboard_api.py +++ b/seqr/views/apis/dashboard_api.py @@ -1,6 +1,3 @@ -""" -APIs used by the main seqr dashboard page -""" from django.db import models from django.db.models.functions import Coalesce @@ -13,15 +10,6 @@ @login_and_policies_required def dashboard_page_data(request): - """Returns a JSON object containing information used by the dashboard page: - :: - - json_response = { - 'user': {..}, - 'familiesByGuid': {..}, - 'individualsByGuid': {..}, - } - """ projects_by_guid = _get_projects_json(request.user) project_categories_by_guid = _retrieve_project_categories_by_guid(projects_by_guid.keys()) @@ -81,17 +69,6 @@ def _get_projects_json(user): def _retrieve_project_categories_by_guid(project_guids): - """Retrieves project categories from the database, and returns a 'project_categories_by_guid' dictionary, - while also adding a 'projectCategoryGuids' attribute to each project dict in 'projects_by_guid'. - - Args: - projects_by_guid: Dictionary that maps each project's GUID to a dictionary of key-value - pairs representing attributes of that project. - - Returns: - Dictionary that maps each category's GUID to a dictionary of key-value pairs representing - attributes of that category. - """ if len(project_guids) == 0: return {} diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py index d6acdb0104..d3684a5ef6 100644 --- a/seqr/views/apis/family_api.py +++ b/seqr/views/apis/family_api.py @@ -1,6 +1,3 @@ -""" -APIs used to retrieve and modify Individual fields -""" import json from collections import defaultdict from django.contrib.auth.models import User @@ -145,12 +142,6 @@ def family_variant_tag_summary(request, family_guid): @login_and_policies_required def edit_families_handler(request, project_guid): - """Edit or one or more Family records. - - Args: - project_guid (string): GUID of project that contains these individuals. - """ - project = get_project_and_check_pm_permissions(project_guid, request.user) request_json = json.loads(request.body) @@ -201,12 +192,6 @@ def edit_families_handler(request, project_guid): @login_and_policies_required def delete_families_handler(request, project_guid): - """Edit or delete one or more Individual records. - - Args: - project_guid (string): GUID of project that contains these individuals. - """ - project = get_project_and_check_pm_permissions(project_guid, request.user, override_permission_func=external_anvil_project_can_edit) request_json = json.loads(request.body) @@ -238,12 +223,6 @@ def delete_families_handler(request, project_guid): @login_and_policies_required def update_family_fields_handler(request, family_guid): - """Updates the specified field in the Family model. - - Args: - family_guid (string): GUID of the family. - """ - family = Family.objects.get(guid=family_guid) # check permission - can be edited by anyone with access to the project @@ -266,11 +245,6 @@ def _set_display_name(family_json, family_model): @login_and_policies_required def update_family_assigned_analyst(request, family_guid): - """Updates the specified field in the Family model. - - Args: - family_guid (string): GUID of the family. - """ family = Family.objects.get(guid=family_guid) # assigned_analyst can be edited by anyone with access to the project check_project_permissions(family.project, request.user, can_edit=False) @@ -298,13 +272,6 @@ def update_family_assigned_analyst(request, family_guid): @login_and_policies_required def update_family_analysed_by(request, family_guid): - """Updates the specified field in the Family model. - - Args: - family_guid (string): GUID of the family. - field_name (string): Family model field name to update - """ - family = Family.objects.get(guid=family_guid) # analysed_by can be edited by anyone with access to the project check_project_permissions(family.project, request.user, can_edit=False) @@ -319,12 +286,6 @@ def update_family_analysed_by(request, family_guid): @login_and_policies_required def update_family_pedigree_image(request, family_guid): - """Updates the specified field in the Family model. - - Args: - family_guid (string): GUID of the family. - """ - family = Family.objects.get(guid=family_guid) # check permission @@ -388,6 +349,7 @@ def receive_families_table_handler(request, project_guid): Args: request (object): Django request object project_guid (string): project GUID + """ project = get_project_and_check_pm_permissions(project_guid, request.user) diff --git a/seqr/views/apis/feature_updates_api.py b/seqr/views/apis/feature_updates_api.py index 140b650182..e8e7940340 100644 --- a/seqr/views/apis/feature_updates_api.py +++ b/seqr/views/apis/feature_updates_api.py @@ -19,10 +19,10 @@ def fetch_feature_updates(): def get_feature_updates(request): - """ - Fetches the feature-updates GitHub Discussion Atom feed, converts feed entries into markdown, and returns + """Fetches the feature-updates GitHub Discussion Atom feed, converts feed entries into markdown, and returns markdown and information for each feed entry. """ + entries = [] for entry in fetch_feature_updates(): # Atom feeds can have multiple content elements per feed entry diff --git a/seqr/views/apis/project_api.py b/seqr/views/apis/project_api.py index 4641d331c9..db0eb69245 100644 --- a/seqr/views/apis/project_api.py +++ b/seqr/views/apis/project_api.py @@ -1,7 +1,3 @@ -""" -APIs for updating project metadata, as well as creating or deleting projects -""" - import json from collections import defaultdict from django.contrib.postgres.aggregates import ArrayAgg @@ -39,19 +35,6 @@ @pm_required def create_project_handler(request): - """Create a new project. - - HTTP POST - Request body - should contain json params: - name: Project name - description: Project description - - Response body - will be json with the following structure, representing the ,created project: - { - 'projectsByGuid': { : { ... ... } } - } - - """ request_json = json.loads(request.body) required_fields = ['name', 'genomeVersion'] @@ -92,29 +75,6 @@ def _is_valid_anvil_workspace(request_json, user): @login_and_policies_required def update_project_handler(request, project_guid): - """Update project metadata - including one or more of these fields: name, description - - Args: - project_guid (string): GUID of the project that should be updated - - HTTP POST - Request body - should contain the following json structure: - { - 'form' : { - 'name': , - 'description': , - } - } - - Response body - will contain the following structure, representing the updated project: - { - 'projectsByGuid': { - : { ... ... } - } - } - - """ - project = Project.objects.get(guid=project_guid) check_project_permissions(project, request.user, can_edit=True) @@ -156,12 +116,6 @@ def update_project_workspace(request, project_guid): @login_and_policies_required def delete_project_handler(request, project_guid): - """Delete project - request handler. - - Args: - project_guid (string): GUID of the project to delete - """ - _delete_project(project_guid, request.user) return create_json_response({ @@ -173,12 +127,6 @@ def delete_project_handler(request, project_guid): @login_and_policies_required def project_page_data(request, project_guid): - """ - Returns a JSON object containing basic project information - - Args: - project_guid (string): GUID of the Project to retrieve data for. - """ project = get_project_and_check_permissions(project_guid, request.user) update_project_from_json(project, {'last_accessed_date': timezone.now()}, request.user) return create_json_response({ @@ -421,12 +369,6 @@ def subscribe_project_notifications(request, project_guid): def _delete_project(project_guid, user): - """Delete project. - - Args: - project_guid (string): GUID of the project to delete - user (object): Django ORM model for the user - """ project = Project.objects.get(guid=project_guid) check_user_created_object_permissions(project, user) diff --git a/seqr/views/apis/project_categories_api.py b/seqr/views/apis/project_categories_api.py index 976bf2bee1..55f15ea3c2 100644 --- a/seqr/views/apis/project_categories_api.py +++ b/seqr/views/apis/project_categories_api.py @@ -1,4 +1,3 @@ -"""APIs for setting Project categories""" import json from seqr.models import Project, ProjectCategory @@ -10,31 +9,6 @@ @login_and_policies_required def update_project_categories_handler(request, project_guid): - """Update ProjectCategories for the given project. - - Args: - project_guid (string): GUID of the project that should be updated - - HTTP POST - Request body - should contain the following json structure: - { - 'form' : { - 'categories': a list of category GUIDs for the categories assigned to the given project - } - } - - Response body - will contain the following structure, representing the updated project, - as well all categories in seqr: - { - 'projectsByGuid': { - : { ... ... } - } - 'projectCategoriesByGuid': { - : { ... ... } - : { ... ... } - } - } - """ project = Project.objects.get(guid=project_guid) # check permissions @@ -58,14 +32,6 @@ def update_project_categories_handler(request, project_guid): def _update_project_categories(project, user, category_guids): - """Updates the stored categories for the given project. - - Args: - project (project): Django Project model - user (User): Django User model - category_guids (set): set of category GUIDs to apply to the given project - """ - category_guids = set(category_guids) project_categories_by_guid = {} # keep track of new and removed categories so client can be updated. diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index 0d78858e59..61f1748532 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -50,8 +50,6 @@ @login_and_policies_required def query_variants_handler(request, search_hash): - """Search variants. - """ page = int(request.GET.get('page') or 1) per_page = int(request.GET.get('per_page') or 100) sort = request.GET.get('sort') or XPOS_SORT_KEY @@ -184,8 +182,6 @@ def _get_exclude_keys(search_hash, user): @login_and_policies_required def query_single_variant_handler(request, variant_id): - """Search variants. - """ families = Family.objects.filter(guid=request.GET.get('familyGuid')) family = families.first() check_project_permissions(family.project, request.user) @@ -404,8 +400,6 @@ def _get_field_value(value, config): @login_and_policies_required def search_context_handler(request): - """Search variants. - """ response = _get_saved_searches(request.user) context = json.loads(request.body) diff --git a/seqr/views/utils/export_utils.py b/seqr/views/utils/export_utils.py index df0d51cfaf..92992a6209 100644 --- a/seqr/views/utils/export_utils.py +++ b/seqr/views/utils/export_utils.py @@ -24,8 +24,10 @@ def export_table(filename_prefix, header, rows, file_format='tsv', titlecase_hea header (list): List of column names rows (list): List of rows, where each row is a list of column values file_format (string): "tsv", "xls", or "json" + Returns: Django HttpResponse object with the table data as an attachment. + """ for i, row in enumerate(rows): diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index 8c6c296fec..b1fcaf7209 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -1,6 +1,3 @@ -""" -APIs for retrieving, updating, creating, and deleting Individual records -""" from collections import defaultdict from clickhouse_search.models.postgres_dicts import SexDict @@ -24,19 +21,6 @@ def _get_record_individual_id(record): def add_or_update_individuals_and_families(project, individual_records, user, get_update_json=True, get_updated_individual_db_ids=False, get_created_counts=False, allow_features_update=False, skip_gt_stats_rebuild=False): - """ - Add or update individual and family records in the given project. - - Args: - project (object): Django ORM model for the project to add families to - individual_records (list): A list of JSON records representing individuals. See - the return value of pedigree_info_utils#convert_fam_file_rows_to_json(..) - user (object): current user model - - Return: - 3-tuple: updated Individual models, updated Family models, and updated FamilyNote models - - """ updated_family_ids = set() updated_individuals = set() updated_affected = set() @@ -180,15 +164,6 @@ def _update_from_record(record, user, families_by_id, individual_lookup, updated def delete_individuals(project, individual_guids, user): - """Delete one or more individuals - - Args: - project (object): Django ORM model for project - individual_guids (list): GUIDs of individuals to delete - - Returns: - list: Family objects for families with deleted individuals - """ errors, individuals_to_delete = check_project_individuals_deletable(project, individual_guids=individual_guids) if errors: raise ErrorsWarningsException(errors) diff --git a/seqr/views/utils/json_utils.py b/seqr/views/utils/json_utils.py index 60cd6b77d0..78bad25f44 100644 --- a/seqr/views/utils/json_utils.py +++ b/seqr/views/utils/json_utils.py @@ -17,15 +17,6 @@ def default(self, o): def create_json_response(obj, **kwargs): - """Encodes the give object into json and create a django JsonResponse object with it. - - Args: - obj (object): json response object - **kwargs: any addition args to pass to the JsonResponse constructor - Returns: - JsonResponse - """ - dumps_params = { 'sort_keys': True, 'indent': 4, @@ -40,7 +31,6 @@ def create_json_response(obj, **kwargs): def _to_camel_case(snake_case_str): - """Convert snake_case string to CamelCase""" if not CAMEL_CASE_MAP.get(snake_case_str): converted = snake_case_str.replace('_', ' ').title().replace(' ', '') CAMEL_CASE_MAP[snake_case_str] = converted[0].lower() + converted[1:] @@ -48,13 +38,10 @@ def _to_camel_case(snake_case_str): def _to_title_case(snake_case_str): - """Convert snake_case string to Title Case""" - components = snake_case_str.split('_') return " ".join(x.title() for x in components) def _to_snake_case(camel_case_str): - """Convert CamelCase string to snake_case (from https://gist.github.com/jaytaylor/3660565)""" - + # from https://gist.github.com/jaytaylor/3660565 return re.sub('([A-Z])', '_\\1', camel_case_str).lower().lstrip('_') diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index cd4cab9d9c..353077c447 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -1,7 +1,3 @@ -""" -Utility functions for converting Django ORM object to JSON -""" - from collections import defaultdict from django.contrib.postgres.aggregates import ArrayAgg from django.db.models import prefetch_related_objects, Count, Value, F, Q, CharField, Case, When @@ -41,18 +37,6 @@ def _get_model_json_fields(model_class, user, is_analyst, additional_model_field def _get_json_for_models(models, nested_fields=None, user=None, is_analyst=None, process_result=None, guid_key=None, additional_model_fields=None): - """Returns an array JSON representations of the given models. - - Args: - models (array): Array of django models - user (object): Django User object for determining whether to include restricted/internal-only fields - nested_fields (array): Optional array of fields to get from the model that are nested on related objects - process_result (lambda): Optional function to post-process a given model json - guid_key (string): Optional key to use for the model's guid - Returns: - array: json objects - """ - if not models: return [] @@ -93,14 +77,6 @@ def _get_json_for_models(models, nested_fields=None, user=None, is_analyst=None, def _get_json_for_model(model, get_json_for_models=_get_json_for_models, **kwargs): - """Helper function to return a JSON representations of the given model. - - Args: - model (object): Django models - get_json_for_models (lambda): Function used to determine the json for an array of the given model - Returns: - object: json object - """ return get_json_for_models([model], **kwargs)[0] @@ -171,14 +147,6 @@ def get_json_for_current_user(user): def get_json_for_projects(projects, user=None, is_analyst=None, add_project_category_guids_field=True, add_permissions=False): - """Returns JSON representation of the given Projects. - - Args: - projects (object array): Django models for the projects - user (object): Django User object for determining whether to include restricted/internal-only fields - Returns: - dict: json object - """ def _process_result(result, project): result.update({ 'isMmeEnabled': result['isMmeEnabled'] and not result['isDemo'], @@ -198,14 +166,6 @@ def _process_result(result, project): def _get_json_for_project(project, user, **kwargs): - """Returns JSON representation of the given Project. - - Args: - project (object): Django model for the project - user (object): Django User object for determining whether to include restricted/internal-only fields - Returns: - dict: json object - """ return _get_json_for_model(project, get_json_for_models=get_json_for_projects, user=user, add_permissions=True, **kwargs) @@ -323,14 +283,6 @@ def _get_sample_json_kwargs(project_guid=None, family_guid=None, individual_guid def get_json_for_samples(samples, **kwargs): - """Returns a JSON representation of the given list of Samples. - - Args: - samples (array): array of django models for the Samples. - Returns: - array: array of json objects - """ - return get_json_for_queryset(samples, **_get_sample_json_kwargs(**kwargs)) @@ -343,27 +295,10 @@ def get_json_for_datasets(datasets, project_guid): def get_json_for_sample(sample, **kwargs): - """Returns a JSON representation of the given Sample. - - Args: - sample (object): Django model for the Sample. - Returns: - dict: json object - """ - return _get_json_for_model(sample, **_get_sample_json_kwargs(**kwargs)) def get_json_for_analysis_groups(analysis_groups, project_guid=None, skip_nested=False, is_dynamic=False, **kwargs): - """Returns a JSON representation of the given list of AnalysisGroups. - - Args: - analysis_groups (array): array of django models for the AnalysisGroups. - project_guid (string): An optional field to use as the projectGuid instead of querying the DB - Returns: - array: array of json objects - """ - def _process_result(result, group): result.update({ 'familyGuids': [f.guid for f in group.families.all()] @@ -384,14 +319,6 @@ def _process_result(result, group): def get_json_for_analysis_group(analysis_group, **kwargs): - """Returns a JSON representation of the given AnalysisGroup. - - Args: - analysis_group (object): Django model for the AnalysisGroup. - project_guid (string): An optional field to use as the projectGuid instead of querying the DB - Returns: - dict: json object - """ return _get_json_for_model(analysis_group, get_json_for_models=get_json_for_analysis_groups, **kwargs) @@ -554,36 +481,14 @@ def get_json_for_discovery_tags(variants, user): def get_json_for_variant_note(note): - """Returns a JSON representation of the given variant note. - - Args: - note (object): Django model for the VariantNote. - Returns: - dict: json object - """ return _get_json_for_model(note, guid_key='noteGuid') def get_json_for_gene_notes(notes, user): - """Returns a JSON representation of the given gene note. - - Args: - note (object): Django model for the GeneNote. - Returns: - dict: json object - """ - return _get_json_for_models(notes, user=user, guid_key='noteGuid') def get_json_for_gene_notes_by_gene_id(gene_ids, user): - """Returns a JSON representation of the gene notes for the given gene ids. - - Args: - note (object): Django model for the GeneNote. - Returns: - dict: json object - """ notes_by_gene_id = defaultdict(list) for note in get_json_for_gene_notes(GeneNote.objects.filter(gene_id__in=gene_ids), user): notes_by_gene_id[note['geneId']].append(note) @@ -619,13 +524,6 @@ def get_json_for_locus_lists(locus_lists, user, include_metadata=True, additiona def get_json_for_locus_list(locus_list, user): - """Returns a JSON representation of the given LocusList. - - Args: - locus_list (object): LocusList django model. - Returns: - dict: json object - """ result = _get_json_for_model(locus_list, user=user) intervals = _get_json_for_models(locus_list.locuslistinterval_set.all()) @@ -663,7 +561,6 @@ def get_json_for_project_collaborator_groups(project): def get_json_for_project_collaborator_list(user, project): - """Returns a JSON representation of the collaborators in the given project""" collaborator_list = list( get_project_collaborators_by_username( user, project, fields=['username', 'email', 'display_name'], include_permissions=True, @@ -674,7 +571,6 @@ def get_json_for_project_collaborator_list(user, project): def get_project_collaborators_by_username(user, project, fields, include_permissions=False, expand_user_groups=False): - """Returns a JSON representation of the collaborators in the given project""" collaborators = {} if not anvil_enabled(): if expand_user_groups: diff --git a/seqr/views/utils/pedigree_info_utils.py b/seqr/views/utils/pedigree_info_utils.py index b87573e76a..6fd9c94f36 100644 --- a/seqr/views/utils/pedigree_info_utils.py +++ b/seqr/views/utils/pedigree_info_utils.py @@ -1,4 +1,3 @@ -"""Utilities for parsing .fam files or other tables that describe individual pedigree structure.""" import difflib import os import json @@ -25,23 +24,6 @@ def parse_pedigree_table(parsed_file, filename, user, project): - """Validates and parses pedigree information from a .fam, .tsv, or Excel file. - - Args: - parsed_file (array): The parsed output from the raw file. - filename (string): The original filename - used to determine the file format based on the suffix. - user (User): Django User object - project (Project): Django Project object - - Return: - A 3-tuple that contains: - ( - json_records (list): list of dictionaries, with each dictionary containing info about - one of the individuals in the input data - errors (list): list of error message strings - warnings (list): list of warning message strings - ) - """ header_string = str(parsed_file[0]) is_merged_pedigree_sample_manifest = "do not modify" in header_string.lower() and "Broad" in header_string if is_merged_pedigree_sample_manifest: @@ -153,32 +135,6 @@ def parse_hpo_terms(hpo_term_string): def _convert_fam_file_rows_to_json(column_map, rows, required_columns=None, update_features=False): - """Parse the values in rows and convert them to a json representation. - - Args: - rows (list): a list of rows where each row is a list of strings corresponding to values in the table - - Returns: - list: a list of dictionaries with each dictionary being a json representation of a parsed row. - For example: - { - 'familyId': family_id, - 'individualId': individual_id, - 'paternalId': paternal_id, - 'maternalId': maternal_id, - 'sex': sex, - 'affected': affected, - 'notes': notes, - 'codedPhenotype': , - 'hpoTermsPresent': [...], - 'hpoTermsAbsent': [...], - 'fundingSource': [...], - 'caseReviewStatus': [...], - } - - Raises: - ValueError: if there are unexpected values or row sizes - """ required_columns = [JsonConstants.FAMILY_ID_COLUMN, JsonConstants.INDIVIDUAL_ID_COLUMN] + (required_columns or []) missing_cols = [_to_title_case(_to_snake_case(col)) for col in set(required_columns) - set(column_map.values())] if update_features and JsonConstants.FEATURES not in column_map.values(): @@ -252,19 +208,6 @@ def _format_value(value, column): def validate_fam_file_records(project, records, errors=None, clear_invalid_values=False, update_features=False, related_guids=None, search_dataset_type=None, validate_expected_samples=None): - """Basic validation such as checking that parents have the same family id as the child, etc. - - Args: - records (list): a list of dictionaries (see return value of #process_rows). - - Returns: - dict: json representation of any errors, warnings, or info messages: - { - 'errors': ['error text1', 'error text2', ...], - 'warnings': ['warning text1', 'warning text2', ...], - 'info': ['info message', ...], - } - """ records_by_id = {r[JsonConstants.PREVIOUS_INDIVIDUAL_ID_COLUMN]: r for r in records if r.get(JsonConstants.PREVIOUS_INDIVIDUAL_ID_COLUMN)} records_by_id.update({r[JsonConstants.INDIVIDUAL_ID_COLUMN]: r for r in records}) @@ -452,13 +395,6 @@ def _validate_parent(row, parent_id_type, parent_id_field, parent_guid_field, ex def _is_header_row(row): - """Checks if the 1st row of a table is a header row - - Args: - row (string): 1st row of a table - Returns: - True if it's a header row rather than data - """ row = row.lower() if "family" in row and ("indiv" in row or "participant" in row): return True @@ -488,18 +424,6 @@ def _parse_merged_pedigree_sample_manifest_rows(rows): def _parse_merged_pedigree_sample_manifest_format(rows, project): - """Does post-processing of rows from Broad's sample manifest + pedigree table format. Expected columns are: - - Kit ID, Well Position, Sample ID, Family ID, Collaborator Participant ID, Collaborator Sample ID, - Paternal Sample ID, Maternal ID, Gender, Affected Status, Volume, Concentration, Notes, Coded Phenotype, - Data Use Restrictions - - Args: - rows (list): A list of lists where each list contains values from each column in the table. - - Returns: - 3-tuple: rows, sample_manifest_rows, kit_id - """ c = MergedPedigreeSampleManifestConstants kit_id = rows[0][c.KIT_ID_COLUMN] diff --git a/seqr/views/utils/permissions_utils.py b/seqr/views/utils/permissions_utils.py index 5ec013e8f9..d5a03f97cd 100644 --- a/seqr/views/utils/permissions_utils.py +++ b/seqr/views/utils/permissions_utils.py @@ -112,14 +112,6 @@ def get_internal_projects(): def get_project_and_check_permissions(project_guid, user, **kwargs): - """Retrieves Project with the given guid after checking that the given user has permission to - retrieve the given project. - - Args: - project_guid (string): GUID of project to retrieve - user (User): Django User object - can_edit (bool): If user need edit permission - """ return _get_project_and_check_permissions(project_guid, user, check_project_permissions, **kwargs) def get_project_and_check_pm_permissions(project_guid, user, override_permission_func=None): diff --git a/seqr/views/utils/terra_api_utils.py b/seqr/views/utils/terra_api_utils.py index 05c9f3ae5c..06fff506d0 100644 --- a/seqr/views/utils/terra_api_utils.py +++ b/seqr/views/utils/terra_api_utils.py @@ -1,5 +1,3 @@ -"""Provide python bindings for the AnVIL Terra API.""" - from datetime import datetime import google.auth.transport.requests import json @@ -28,33 +26,17 @@ class TerraAPIException(Exception): def __init__(self, message, status_code): - """ - Custom Exception to capture Terra API call failures - - :param message: error message - :param status_code: the status code associated with the failed request - """ super(TerraAPIException, self).__init__(message) self.status_code = status_code class TerraNotFoundException(TerraAPIException): def __init__(self, message): - """ - Custom Exception to capture Terra API calls that fail with 404 Not Found - - :param message: error message - """ super(TerraNotFoundException, self).__init__(message, 404) class TerraRefreshTokenFailedException(TerraAPIException): def __init__(self, message): - """ - Custom Exception to capture refresh token failures. Maps to 401 error to redirect user to login - - :param message: error message - """ super(TerraRefreshTokenFailedException, self).__init__(message, 401) @@ -178,15 +160,6 @@ def _user_anvil_call(method, path, user, **kwargs): def list_anvil_workspaces(user): - """Get all the workspaces accessible by the logged-in user. - - :param - user (User model): who's credentials will be used to access AnVIL - :return - A list of workspaces that the user has access (OWNER, WRITER, or READER). Each of the workspace has - its name and namespace. - - """ return _user_anvil_call( 'get', 'api/workspaces?fields=public,workspace.name,workspace.namespace', user, cache_time=TERRA_WORKSPACE_CACHE_EXPIRE_SECONDS, diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index 02335af03e..233b40d66e 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -27,7 +27,7 @@ def _databases_support_transactions(cls): @classmethod def _rollback_atomics(cls, atomics): - """Django testcases asssume either all database support transactions or none do. This properly cleans up transaction blocks on a per-db basis""" + # Django testcases asssume either all database support transactions or none do. This properly cleans up transaction blocks on a per-db basis for db_name in reversed(cls._databases_names()): if connections[db_name].features.supports_transactions: transaction.set_rollback(True, using=db_name) @@ -170,15 +170,6 @@ def login_data_manager_user(self): def _check_login(self, url, permission_level, request_data=None, login_redirect_url='/api/login-required-error', policy_redirect_url='/api/policy-required-error', permission_denied_error=403): - """For integration tests of django views that can only be accessed by a logged-in user, - the 1st step is to authenticate. This function checks that the given url redirects requests - if the user isn't logged-in, and then authenticates a test user. - - Args: - test_case (object): the django.TestCase or unittest.TestCase object - url (string): The url of the django view being tested. - permission_level (string): what level of permission this url requires - """ # check that it redirects if you don't login parsed_url = urlparse(url) next_query = quote_plus('?{}'.format(parsed_url.query)) if parsed_url.query else '' diff --git a/wsgi.py b/wsgi.py index 1fd7e20bda..a8f0b255bc 100644 --- a/wsgi.py +++ b/wsgi.py @@ -1,18 +1,3 @@ -""" -WSGI config for xbrowse_server project. - -This module contains the WSGI application used by Django's development server -and any production WSGI deployments. It should expose a module-level variable -named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover -this application via the ``WSGI_APPLICATION`` setting. - -Usually you will have the standard Django WSGI application here, but it also -might make sense to replace the whole Django WSGI application with a custom one -that later delegates to the Django one. For example, you could introduce WSGI -middleware here, or combine a Django application with an application of another -framework. - -""" import os os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") From 0eb632ccb6d0fe7a597e01615a09d5ab7e14b5c6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 May 2026 11:52:45 -0400 Subject: [PATCH 071/119] nosec for tests --- matchmaker/views/matchmaker_api_tests.py | 4 ++-- seqr/utils/social_auth_pipeline_tests.py | 4 ++-- seqr/views/apis/auth_api_tests.py | 6 +++--- seqr/views/apis/users_api_tests.py | 2 +- seqr/views/utils/terra_api_utils_tests.py | 2 +- vlm/test_vlm.py | 6 +++--- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/matchmaker/views/matchmaker_api_tests.py b/matchmaker/views/matchmaker_api_tests.py index 267c765fb8..2b012d6ddb 100644 --- a/matchmaker/views/matchmaker_api_tests.py +++ b/matchmaker/views/matchmaker_api_tests.py @@ -157,8 +157,8 @@ MOCK_SLACK_TOKEN = 'xoxp-123' # nosec MOCK_NODES_BY_NAME = { - 'Node A': {'name': 'Node A', 'token': 'abc', 'url': 'http://node_a.com/match'}, - 'Node B': {'name': 'Node B', 'token': 'xyz', 'url': 'http://node_b.mme.org/api'}, + 'Node A': {'name': 'Node A', 'token': 'abc', 'url': 'http://node_a.com/match'}, # nosec + 'Node B': {'name': 'Node B', 'token': 'xyz', 'url': 'http://node_b.mme.org/api'}, # nosec } class EmailException(Exception): diff --git a/seqr/utils/social_auth_pipeline_tests.py b/seqr/utils/social_auth_pipeline_tests.py index bb08aad28f..9208004d85 100644 --- a/seqr/utils/social_auth_pipeline_tests.py +++ b/seqr/utils/social_auth_pipeline_tests.py @@ -15,7 +15,7 @@ class SocialAuthPipelineTest(TestCase): def test_validate_anvil_registration(self, mock_logger): url = TEST_TERRA_API_ROOT_URL + 'register' responses.add(responses.GET, url, status=404) - r = validate_anvil_registration(GoogleOAuth2(), {'access_token': '', 'email': 'test@seqr.org'}) + r = validate_anvil_registration(GoogleOAuth2(), {'access_token': '', 'email': 'test@seqr.org'}) # nosec mock_logger.warning.assert_called_with( 'User test@seqr.org is trying to login without registration on AnVIL. None called Terra API: GET /register got status 404 with reason: Not Found', extra={'user_email': 'test@seqr.org'} @@ -25,7 +25,7 @@ def test_validate_anvil_registration(self, mock_logger): backend = GoogleOAuth2() backend.strategy.session_set('next', '/foo/bar') - r = validate_anvil_registration(backend, {'access_token': '', 'email': 'test@seqr.org'}) + r = validate_anvil_registration(backend, {'access_token': '', 'email': 'test@seqr.org'}) # nosec self.assertEqual(r.url, '/login/error/anvil_registration?next=%2Ffoo%2Fbar') mock_logger.reset_mock() diff --git a/seqr/views/apis/auth_api_tests.py b/seqr/views/apis/auth_api_tests.py index 9d8f28f288..a448373790 100644 --- a/seqr/views/apis/auth_api_tests.py +++ b/seqr/views/apis/auth_api_tests.py @@ -28,7 +28,7 @@ def test_login_view(self): # send login request without an email req_values = { - 'password': 'not_a_password' + 'password': 'not_a_password' # nosec } response = self.client.post(url, content_type='application/json', data=json.dumps(req_values)) @@ -55,7 +55,7 @@ def test_login_view(self): # send login request with unicode character req_values = { 'email': 'test_new_user@instıtute.com', - 'password': 'password123' + 'password': 'password123' # nosec } response = self.client.post(url, content_type='application/json', data=json.dumps(req_values)) self.assertEqual(response.status_code, 401) @@ -74,7 +74,7 @@ def test_login_view(self): # send login request with a privileged account (data manage or superuser) while Google auth is not enabled req_values = { 'email': 'test_data_manager@test.com', - 'password': 'not-a-password' + 'password': 'not-a-password' # nosec } response = self.client.post(url, content_type='application/json', data=json.dumps(req_values)) diff --git a/seqr/views/apis/users_api_tests.py b/seqr/views/apis/users_api_tests.py index 7d7efe64a3..c45f86d892 100644 --- a/seqr/views/apis/users_api_tests.py +++ b/seqr/views/apis/users_api_tests.py @@ -388,7 +388,7 @@ def _test_set_password(self, set_password_url, password): # pylint: disable=argu self.assertEqual(response.reason_phrase, 'Password is required') response = self.client.post(set_password_url, content_type='application/json', data=json.dumps({ - 'userToken': quote_plus(password), 'password': 'password123', 'firstName': 'Test', 'isSuperuser': True})) + 'userToken': quote_plus(password), 'password': 'password123', 'firstName': 'Test', 'isSuperuser': True})) # nosec self.assertEqual(response.status_code, 200) user = User.objects.get(username='test_new_user') diff --git a/seqr/views/utils/terra_api_utils_tests.py b/seqr/views/utils/terra_api_utils_tests.py index 8268bd06ac..f416dc7393 100644 --- a/seqr/views/utils/terra_api_utils_tests.py +++ b/seqr/views/utils/terra_api_utils_tests.py @@ -13,7 +13,7 @@ add_service_account, has_service_account_access, get_anvil_group_members, user_get_anvil_groups GET_WORKSPACE_PATH = 'api/workspaces?fields=public,workspace.name,workspace.namespace' -AUTH_EXTRA_DATA = {"expires": 3599, "auth_time": TOKEN_AUTH_TIME, "token_type": "Bearer", "access_token": "ya29.EXAMPLE"} +AUTH_EXTRA_DATA = {"expires": 3599, "auth_time": TOKEN_AUTH_TIME, "token_type": "Bearer", "access_token": "ya29.EXAMPLE"} # nosec LIST_WORKSPACE_RESPONSE = '[{"accessLevel": "PROJECT_OWNER", "public": false, "workspace": {"attributes": {"description": "Workspace for seqr project"}, "authorizationDomain": [], "bucketName": "fc-237998e6-663d-40b9-bd13-57c3bb6ac593", "createdBy": "test1@test.com", "createdDate": "2020-09-09T15:10:32.816Z", "isLocked": false, "lastModified": "2020-09-09T15:10:32.818Z", "name": "1000 Genomes Demo", "namespace": "my-seqr-billing", "workflowCollectionName": "237998e6-663d-40b9-bd13-57c3bb6ac593", "workspaceId": "237998e6-663d-40b9-bd13-57c3bb6ac593" }, "workspaceSubmissionStats": {"runningSubmissionsCount": 0}},\ {"accessLevel": "READER","public": true, "workspace": {"attributes": {"tag:tags": {"itemsType": "AttributeValue","items": ["differential-expression","tutorial"]},"description": "[DEGenome](https://github.com/eweitz/degenome) transforms differential expression data into inputs for [exploratory genome analysis with Ideogram.js](https://eweitz.github.io/ideogram/differential-expression?annots-url=https://www.googleapis.com/storage/v1/b/degenome/o/GLDS-4_array_differential_expression_ideogram_annots.json). \\n\\nTry the [Notebook tutorial](https://app.terra.bio/#workspaces/degenome/degenome/notebooks/launch/degenome-tutorial.ipynb), where you can step through using DEGenome to analyze expression for mice flown in space!"},"authorizationDomain": [],"bucketName": "fc-2706d493-5fce-4fb2-9993-457c30364a06","createdBy": "test2@test.com","createdDate": "2020-01-14T10:21:14.575Z","isLocked": false,"lastModified": "2020-02-01T13:28:27.309Z","name": "degenome","namespace": "degenome","workflowCollectionName": "2706d493-5fce-4fb2-9993-457c30364a06","workspaceId": "2706d493-5fce-4fb2-9993-457c30364a06"},"workspaceSubmissionStats": {"runningSubmissionsCount": 0}},\ {"accessLevel": "PROJECT_OWNER","public": false, "workspace": {"attributes": {"description": "A workspace for seqr project"},"authorizationDomain": [],"bucketName": "fc-6a048145-c134-4004-a009-42824f826ee8","createdBy": "test3@test.com","createdDate": "2020-09-09T15:12:30.142Z","isLocked": false,"lastModified": "2020-09-09T15:12:30.145Z","name": "seqr-project 1000 Genomes Demo","namespace": "my-seqr-billing","workflowCollectionName": "6a048145-c134-4004-a009-42824f826ee8","workspaceId": "6a048145-c134-4004-a009-42824f826ee8"},"workspaceSubmissionStats": {"runningSubmissionsCount": 0}}]' diff --git a/vlm/test_vlm.py b/vlm/test_vlm.py index 7889cddbeb..9b0a1f02e3 100644 --- a/vlm/test_vlm.py +++ b/vlm/test_vlm.py @@ -32,7 +32,7 @@ def inject_fixtures(self, caplog): async def test_match(self, mocked_responses): mocked_responses.post( - 'https://vlm-auth.us.auth0.com/oauth/token', payload={'access_token': 'test_token'}, repeat=True, + 'https://vlm-auth.us.auth0.com/oauth/token', payload={'access_token': 'test_token'}, repeat=True, # nosec ) mocked_responses.get( f'https://vlm-auth.us.auth0.com/api/v2/clients/{REQUESTER_CLIENT_ID}', @@ -106,7 +106,7 @@ async def test_match(self, mocked_responses): mocked_responses.assert_called_with( 'https://vlm-auth.us.auth0.com/oauth/token', method='POST', - json={'client_id': 'unknown_client_id', 'client_secret': 'unknown_client_secret', 'audience': 'https://vlm-auth.us.auth0.com/api/v2/', 'grant_type': 'client_credentials'}, + json={'client_id': 'unknown_client_id', 'client_secret': 'unknown_client_secret', 'audience': 'https://vlm-auth.us.auth0.com/api/v2/', 'grant_type': 'client_credentials'}, # nosec ) mocked_responses.assert_called_with( f'https://vlm-auth.us.auth0.com/api/v2/clients/{REQUESTER_CLIENT_ID}', @@ -292,7 +292,7 @@ async def test_match_error(self, mocked_responses): self.assertEqual(resp.reason, 'Credential Check Error') self.assertEqual(self._caplog.messages[-3], "Credential Check Error: 400 - {'error': 'invalid_request'}") - mocked_responses.post('https://vlm-auth.us.auth0.com/oauth/token', payload={'access_token': 'test_token'}, repeat=True) + mocked_responses.post('https://vlm-auth.us.auth0.com/oauth/token', payload={'access_token': 'test_token'}, repeat=True) # nosec async with self.client.request('GET', '/vlm/match', headers=headers) as resp: self.assertEqual(resp.status, 403) self.assertEqual(resp.reason, 'Invalid Client ID abc123') From d8bb97bfae850f15fea37e3d8f6e8f990c8944bc Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 May 2026 11:56:39 -0400 Subject: [PATCH 072/119] remove unused imports --- clickhouse_search/models/gt_stats_models.py | 1 - clickhouse_search/models/reference_data_models.py | 2 +- reference_data/management/tests/update_gencc_tests.py | 2 -- seqr/views/apis/family_api_tests.py | 1 - seqr/views/apis/project_api_tests.py | 1 - seqr/views/utils/test_utils.py | 2 +- settings.py | 2 -- 7 files changed, 2 insertions(+), 9 deletions(-) diff --git a/clickhouse_search/models/gt_stats_models.py b/clickhouse_search/models/gt_stats_models.py index 86ccb1e05a..3076b65054 100644 --- a/clickhouse_search/models/gt_stats_models.py +++ b/clickhouse_search/models/gt_stats_models.py @@ -1,5 +1,4 @@ from clickhouse_backend import models -from django.db.models import OneToOneField, CASCADE from clickhouse_search.backend.fields import UInt32FieldDeltaCodecField, DictKeyForeignKey from clickhouse_search.backend.table_models import RefreshableMaterializedView, RefreshableMaterializedViewMeta, \ diff --git a/clickhouse_search/models/reference_data_models.py b/clickhouse_search/models/reference_data_models.py index 901fbafb61..d129f8de8b 100644 --- a/clickhouse_search/models/reference_data_models.py +++ b/clickhouse_search/models/reference_data_models.py @@ -1,5 +1,5 @@ from clickhouse_backend import models -from django.db.models import ForeignKey, OneToOneField, CASCADE, PROTECT +from django.db.models import ForeignKey, PROTECT import requests from clickhouse_search.backend.engines import Join diff --git a/reference_data/management/tests/update_gencc_tests.py b/reference_data/management/tests/update_gencc_tests.py index 2d424b786c..af8444e88d 100644 --- a/reference_data/management/tests/update_gencc_tests.py +++ b/reference_data/management/tests/update_gencc_tests.py @@ -1,5 +1,3 @@ -import mock - from reference_data.models import GenCC from reference_data.management.tests.test_utils import ReferenceDataCommandTestCase diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py index 52ea9b1e1d..6a52e6eaec 100644 --- a/seqr/views/apis/family_api_tests.py +++ b/seqr/views/apis/family_api_tests.py @@ -5,7 +5,6 @@ from copy import deepcopy from datetime import datetime from django.core.files.uploadedfile import SimpleUploadedFile -from django.db import connections from django.urls.base import reverse from matchmaker.models import MatchmakerSubmission diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py index 4155a9b96a..16ce8a0629 100644 --- a/seqr/views/apis/project_api_tests.py +++ b/seqr/views/apis/project_api_tests.py @@ -3,7 +3,6 @@ from copy import deepcopy from datetime import datetime from django.contrib.auth.models import Group -from django.db import connections from django.urls.base import reverse import responses diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index 8b628a1210..5dce5c7854 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -14,7 +14,7 @@ import responses from urllib.parse import quote_plus, urlparse -from seqr.models import Project, SavedVariant, CAN_VIEW, CAN_EDIT +from seqr.models import Project, CAN_VIEW, CAN_EDIT WINDOW_REGEX_TEMPLATE = 'window\.{key}=(?P[^)<]+)' diff --git a/settings.py b/settings.py index 94fc51a8b3..5c3871d8cf 100644 --- a/settings.py +++ b/settings.py @@ -4,8 +4,6 @@ import string import subprocess # nosec -from ssl import create_default_context - import google.auth import google.auth.transport.requests From 2a20af3dc6f0852f7cd37c38665fed58b25b988a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 May 2026 11:59:48 -0400 Subject: [PATCH 073/119] fix super args --- clickhouse_search/managers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 7924553107..e7d7b3071c 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -933,7 +933,7 @@ def table_basename(self): def result_values(self, *args, skip_entry_fields=True, **kwargs): return super().result_values(*args, skip_entry_fields=skip_entry_fields, **kwargs) - def join_annotations(self): + def join_annotations(self, *args, **kwargs): results = super().join_annotations(annotate_xpos=bool(self.entry_model.RANGE_PREDICTIONS)) results = results.annotate( hgmd_join=self._pathogenicity_tuple(self.variant_model.hgmd_join, 'key__hgmd_join', rename_fields={'classification': 'class'}), @@ -1050,7 +1050,7 @@ def _filter_frequency(self, entries, frequencies, **kwargs): def _join_annotations(self, entries): return self._annotate_seqr_pop_expression(entries) - def result_values(self, sample_data=None): + def result_values(self, *args, sample_data=None, **kwargs): entries = self._join_annotations(self) return self._search_call_data(entries, sample_data) From 441e33c874d2feba2ea7e087cf862eb123c9cbc3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 May 2026 13:14:50 -0400 Subject: [PATCH 074/119] update ui fixture data --- ui/shared/components/panel/MatchmakerPanel.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/shared/components/panel/MatchmakerPanel.test.js b/ui/shared/components/panel/MatchmakerPanel.test.js index bdba5540d4..8a30161006 100644 --- a/ui/shared/components/panel/MatchmakerPanel.test.js +++ b/ui/shared/components/panel/MatchmakerPanel.test.js @@ -12,7 +12,7 @@ test('shallow-render SubmissionGeneVariants without crashing', () => { const store = configureStore()(STATE1) shallow() }) From c3256f9edd70c380b46776e6d57cd71662896386 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 May 2026 13:16:40 -0400 Subject: [PATCH 075/119] unused variable --- seqr/management/commands/tag_seqr_prioritized_variants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index 477bc2851b..a7807a4077 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -501,7 +501,7 @@ def handle(self, *args, **options): return family_new_counts = defaultdict(int) - for family_id, variant_id in new_tag_keys: + for family_id, _ in new_tag_keys: family_new_counts[family_id] += 1 send_project_notification( From a79f92c00128ca89d6b4e119d9c361c24561442f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 May 2026 13:24:20 -0400 Subject: [PATCH 076/119] reduce function complexity --- .../commands/reload_clinvar_all_variants.py | 64 ++++++++++--------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/clickhouse_search/management/commands/reload_clinvar_all_variants.py b/clickhouse_search/management/commands/reload_clinvar_all_variants.py index 8f14c1a289..418e17c3a8 100644 --- a/clickhouse_search/management/commands/reload_clinvar_all_variants.py +++ b/clickhouse_search/management/commands/reload_clinvar_all_variants.py @@ -253,6 +253,39 @@ def extract_variant_info(elem: xml.etree.ElementTree.Element, new_version: str, **props, ) +def parse_clinvar_file(gzipped_file, model_to_batch): + for event, elem in ET.iterparse(gzipped_file, events=('start', 'end')): + # Handle parsing the current date. + if event == 'start' and elem.tag == 'ClinVarVariationRelease': + new_version = elem.attrib['ReleaseDate'] + existing_version_obj = DataVersions.objects.filter(data_model_name='Clinvar').first() + if existing_version_obj: + if existing_version_obj.version == new_version: + logger.info(f'Clinvar ClickHouse tables already successfully updated to {new_version}, gracefully exiting.') + return + logger.info( f'Updating Clinvar ClickHouse tables to {new_version} from {existing_version_obj and existing_version_obj.version}.') + # Drop any currently existing variants in the table that may exist due to a + # previously failed partial run. Note that we validate that the Postgresql existing version + # is present in ClickHouse to account for the situation where Postgresql has an incorrect + # version. + if existing_version_obj and ClinvarAllVariantsSnvIndel.objects.filter( + version=existing_version_obj.version).exists(): + for model in model_to_batch.keys(): + with connections['clickhouse_write'].cursor() as cursor: + cursor.execute( + f"ALTER TABLE `{model._meta.db_table}` DROP PARTITION '{new_version}';" + ) + # Handle parsing variants + if event == 'end' and elem.tag == 'VariationArchive' and new_version: + for obj in extract_variant_info(elem, new_version, unenumerated_value_alerts): + for model, batch in model_to_batch.items(): + if isinstance(obj, model): + batch.append(obj) + if len(batch) == BATCH_SIZE: + model.objects.using('clickhouse_write').bulk_create(batch) + batch.clear() + elem.clear() + class Command(BaseCommand): help = 'Reload all clinvar variants from weekly NCBI xml release' @@ -270,36 +303,7 @@ def handle(self, *args, **options): r.raise_for_status() shutil.copyfileobj(r.raw, tmpfile) with gzip.open(tmpfile.name, 'rb') as gzipped_file: - for event, elem in ET.iterparse(gzipped_file, events=('start', 'end')): - # Handle parsing the current date. - if event == 'start' and elem.tag == 'ClinVarVariationRelease': - new_version = elem.attrib['ReleaseDate'] - existing_version_obj = DataVersions.objects.filter(data_model_name='Clinvar').first() - if existing_version_obj: - if existing_version_obj.version == new_version: - logger.info(f'Clinvar ClickHouse tables already successfully updated to {new_version}, gracefully exiting.') - return - logger.info(f'Updating Clinvar ClickHouse tables to {new_version} from {existing_version_obj and existing_version_obj.version}.') - # Drop any currently existing variants in the table that may exist due to a - # previously failed partial run. Note that we validate that the Postgresql existing version - # is present in ClickHouse to account for the situation where Postgresql has an incorrect - # version. - if existing_version_obj and ClinvarAllVariantsSnvIndel.objects.filter(version=existing_version_obj.version).exists(): - for model in model_to_batch.keys(): - with connections['clickhouse_write'].cursor() as cursor: - cursor.execute( - f"ALTER TABLE `{model._meta.db_table}` DROP PARTITION '{new_version}';" - ) - # Handle parsing variants - if event == 'end' and elem.tag == 'VariationArchive' and new_version: - for obj in extract_variant_info(elem, new_version, unenumerated_value_alerts): - for model, batch in model_to_batch.items(): - if isinstance(obj, model): - batch.append(obj) - if len(batch) == BATCH_SIZE: - model.objects.using('clickhouse_write').bulk_create(batch) - batch.clear() - elem.clear() + parse_clinvar_file(gzipped_file, model_to_batch) for model, batch in model_to_batch.items(): if batch: From 608cb45f4ad892a47238959d84bca72a5528914b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 May 2026 13:51:50 -0400 Subject: [PATCH 077/119] reduce function complexity --- clickhouse_search/managers.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index e7d7b3071c..59bb3332fa 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -620,12 +620,30 @@ def _filter_annotations(self, results, *args, exclude=None, pathogenicity=None, return results + def _annotations_field_filters(self): + return { + MOTIF_FEATURES_KEY: lambda value: Q(**{ + f'sorted_{MOTIF_FEATURES_KEY}_consequences__array_exists': {'consequenceTerms': (value, 'hasAny({value}, {field})')}, + }) if hasattr(self.model, f'sorted_{MOTIF_FEATURES_KEY}_consequences') else None, + REGULATORY_FEATURES_KEY: lambda value: Q(**{ + f'sorted_{REGULATORY_FEATURES_KEY}_consequences__array_exists': {'consequenceTerms': (value, 'hasAny({value}, {field})')}, + }) if hasattr(self.model, f'sorted_{REGULATORY_FEATURES_KEY}_consequences') else None, + SCREEN_KEY: lambda value: Q(screen__in=value) if self.model.SCREEN_DICT else None, + SPLICE_AI_FIELD: lambda value: Q(preds__0__gte=float(value)) if value and SPLICE_AI_FIELD in self.entry_model.PREDICTIONS else None, + 'mitomap_pathogenic': lambda value: Q(mitomapPathogenic=value) if self.has_annotation('mitomapPathogenic') else None, + } + def _parse_annotation_filters(self, annotations, pathogenicity): filter_qs = [] allowed_consequences = [] transcript_field_filters = {} + annotations_field_filters = self._annotations_field_filters() for field, value in annotations.items(): - if field == UTR_ANNOTATOR_KEY: + if field in annotations_field_filters: + filter_q = annotations_field_filters[field] + if filter_q: + filter_qs.append(filter_q) + elif field == UTR_ANNOTATOR_KEY: if value: transcript_field_filters['fiveutrConsequence'] = (value, 'hasAny({value}, [{field}])') elif field == EXTENDED_SPLICE_KEY: @@ -634,21 +652,6 @@ def _parse_annotation_filters(self, annotations, pathogenicity): value = [c for c in value if c != EXTENDED_SPLICE_REGION_CONSEQUENCE] if value: allowed_consequences += value - elif field in [MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY]: - filter_field = f'sorted_{field}_consequences' - if hasattr(self.model, filter_field): - filter_qs.append(Q(**{ - f'{filter_field}__array_exists': {'consequenceTerms': (value, 'hasAny({value}, {field})')}, - })) - elif field == SCREEN_KEY: - if self.model.SCREEN_DICT: - filter_qs.append(Q(screen__in=value)) - elif field == 'mitomap_pathogenic': - if self.has_annotation('mitomapPathogenic'): - filter_qs.append(Q(mitomapPathogenic=value)) - elif field == SPLICE_AI_FIELD: - if value and SPLICE_AI_FIELD in self.entry_model.PREDICTIONS: - filter_qs.append(Q(preds__0__gte=float(value))) elif field not in SV_ANNOTATION_TYPES: allowed_consequences += value From 43f52c52b3be0d172de33904a72c2154d180a17d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 May 2026 13:55:53 -0400 Subject: [PATCH 078/119] fix missing var --- .../management/commands/reload_clinvar_all_variants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clickhouse_search/management/commands/reload_clinvar_all_variants.py b/clickhouse_search/management/commands/reload_clinvar_all_variants.py index 418e17c3a8..f1c612fe8d 100644 --- a/clickhouse_search/management/commands/reload_clinvar_all_variants.py +++ b/clickhouse_search/management/commands/reload_clinvar_all_variants.py @@ -253,7 +253,7 @@ def extract_variant_info(elem: xml.etree.ElementTree.Element, new_version: str, **props, ) -def parse_clinvar_file(gzipped_file, model_to_batch): +def parse_clinvar_file(gzipped_file, model_to_batch, unenumerated_value_alerts): for event, elem in ET.iterparse(gzipped_file, events=('start', 'end')): # Handle parsing the current date. if event == 'start' and elem.tag == 'ClinVarVariationRelease': @@ -303,7 +303,7 @@ def handle(self, *args, **options): r.raise_for_status() shutil.copyfileobj(r.raw, tmpfile) with gzip.open(tmpfile.name, 'rb') as gzipped_file: - parse_clinvar_file(gzipped_file, model_to_batch) + parse_clinvar_file(gzipped_file, model_to_batch, unenumerated_value_alerts) for model, batch in model_to_batch.items(): if batch: From 45cc73cf4ec0a21c4482aab6df880b820b402d6a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 May 2026 13:59:31 -0400 Subject: [PATCH 079/119] fix missing var --- .../management/commands/reload_clinvar_all_variants.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clickhouse_search/management/commands/reload_clinvar_all_variants.py b/clickhouse_search/management/commands/reload_clinvar_all_variants.py index f1c612fe8d..d9088aca95 100644 --- a/clickhouse_search/management/commands/reload_clinvar_all_variants.py +++ b/clickhouse_search/management/commands/reload_clinvar_all_variants.py @@ -253,12 +253,11 @@ def extract_variant_info(elem: xml.etree.ElementTree.Element, new_version: str, **props, ) -def parse_clinvar_file(gzipped_file, model_to_batch, unenumerated_value_alerts): +def parse_clinvar_file(gzipped_file, existing_version_obj, model_to_batch, unenumerated_value_alerts): for event, elem in ET.iterparse(gzipped_file, events=('start', 'end')): # Handle parsing the current date. if event == 'start' and elem.tag == 'ClinVarVariationRelease': new_version = elem.attrib['ReleaseDate'] - existing_version_obj = DataVersions.objects.filter(data_model_name='Clinvar').first() if existing_version_obj: if existing_version_obj.version == new_version: logger.info(f'Clinvar ClickHouse tables already successfully updated to {new_version}, gracefully exiting.') @@ -297,13 +296,14 @@ def handle(self, *args, **options): } unenumerated_value_alerts = [] new_version = None + existing_version_obj = DataVersions.objects.filter(data_model_name='Clinvar').first() with tempfile.TemporaryDirectory() as tmpdir: with requests.get(WEEKLY_XML_RELEASE, stream=True, timeout=10) as r, \ tempfile.NamedTemporaryFile(dir=tmpdir, delete=False) as tmpfile: r.raise_for_status() shutil.copyfileobj(r.raw, tmpfile) with gzip.open(tmpfile.name, 'rb') as gzipped_file: - parse_clinvar_file(gzipped_file, model_to_batch, unenumerated_value_alerts) + parse_clinvar_file(gzipped_file, existing_version_obj, model_to_batch, unenumerated_value_alerts) for model, batch in model_to_batch.items(): if batch: From 373830ab81488b7e9bcb63cc2031f8b73d003c26 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 May 2026 14:23:02 -0400 Subject: [PATCH 080/119] fix reload clinvar version --- .../management/commands/reload_clinvar_all_variants.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/clickhouse_search/management/commands/reload_clinvar_all_variants.py b/clickhouse_search/management/commands/reload_clinvar_all_variants.py index d9088aca95..50c2ac478e 100644 --- a/clickhouse_search/management/commands/reload_clinvar_all_variants.py +++ b/clickhouse_search/management/commands/reload_clinvar_all_variants.py @@ -261,7 +261,7 @@ def parse_clinvar_file(gzipped_file, existing_version_obj, model_to_batch, unenu if existing_version_obj: if existing_version_obj.version == new_version: logger.info(f'Clinvar ClickHouse tables already successfully updated to {new_version}, gracefully exiting.') - return + return new_version logger.info( f'Updating Clinvar ClickHouse tables to {new_version} from {existing_version_obj and existing_version_obj.version}.') # Drop any currently existing variants in the table that may exist due to a # previously failed partial run. Note that we validate that the Postgresql existing version @@ -285,6 +285,8 @@ def parse_clinvar_file(gzipped_file, existing_version_obj, model_to_batch, unenu batch.clear() elem.clear() + return new_version + class Command(BaseCommand): help = 'Reload all clinvar variants from weekly NCBI xml release' @@ -295,7 +297,6 @@ def handle(self, *args, **options): ClinvarAllVariantsMito: [], } unenumerated_value_alerts = [] - new_version = None existing_version_obj = DataVersions.objects.filter(data_model_name='Clinvar').first() with tempfile.TemporaryDirectory() as tmpdir: with requests.get(WEEKLY_XML_RELEASE, stream=True, timeout=10) as r, \ @@ -303,7 +304,7 @@ def handle(self, *args, **options): r.raise_for_status() shutil.copyfileobj(r.raw, tmpfile) with gzip.open(tmpfile.name, 'rb') as gzipped_file: - parse_clinvar_file(gzipped_file, existing_version_obj, model_to_batch, unenumerated_value_alerts) + new_version = parse_clinvar_file(gzipped_file, existing_version_obj, model_to_batch, unenumerated_value_alerts) for model, batch in model_to_batch.items(): if batch: From 720f4a31225a82f312494987e8a379136a190a58 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 7 May 2026 14:24:54 -0400 Subject: [PATCH 081/119] fix typo --- clickhouse_search/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index 59bb3332fa..b19f8f2250 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -640,7 +640,7 @@ def _parse_annotation_filters(self, annotations, pathogenicity): annotations_field_filters = self._annotations_field_filters() for field, value in annotations.items(): if field in annotations_field_filters: - filter_q = annotations_field_filters[field] + filter_q = annotations_field_filters[field](value) if filter_q: filter_qs.append(filter_q) elif field == UTR_ANNOTATOR_KEY: From 901c5381b5bb60c44646c79037e6f66b518777ee Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 15:18:55 -0400 Subject: [PATCH 082/119] reload affected dict when status updated --- clickhouse_search/backend/table_models.py | 6 ++++++ seqr/views/apis/individual_api_tests.py | 1 + seqr/views/utils/individual_utils.py | 8 +++++--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/clickhouse_search/backend/table_models.py b/clickhouse_search/backend/table_models.py index c75be39d76..eaf34e4ee9 100644 --- a/clickhouse_search/backend/table_models.py +++ b/clickhouse_search/backend/table_models.py @@ -1,9 +1,13 @@ from clickhouse_backend import models from django.db import connections from django.db.models import Func +import logging from clickhouse_search.backend.fields import NamedTupleField +logger = logging.getLogger(__name__) + + MATERIALIZED_VIEW_META_FIELDS = [ 'to_table', 'source_table', 'source_sql', 'source_url', 'source_url_template', 'column_selects', 'refreshable', 'create_empty', ] @@ -41,6 +45,7 @@ class RefreshableMaterializedView(IncrementalMaterializedView): @classmethod def refresh(cls): + logger.info(f'Refreshing materialized view {cls._meta.db_table}') with connections['clickhouse_write'].cursor() as cursor: cursor.execute(f'SYSTEM START VIEW "{cls._meta.db_table}"') cursor.execute(f'SYSTEM REFRESH VIEW "{cls._meta.db_table}"') @@ -89,6 +94,7 @@ def dict_get_expression(cls, *expressions, field_names=None, force_tuple=False, @classmethod def reload(cls): + logger.info(f'Reloading dictionary {cls._meta.db_table}') with connections['clickhouse_write'].cursor() as cursor: cursor.execute(f'SYSTEM RELOAD DICTIONARY "{cls._meta.db_table}"') diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 027b5a8237..dc0e8f248b 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -347,6 +347,7 @@ def test_edit_individuals(self, mock_pm_group): 'updateFields': ['mother'], 'updateType': 'update', }}), + ('Reloading dictionary seqrdb_affected_status_dict', None), ('Triggering rebuild_gt_stats for R0004_non_analyst_project', None), (f'Error Triggering Rebuild Gt Stats: 400 Client Error: Bad Request for url: {TRIGGER_RELOAD_URL}', { 'severity': 'ERROR', diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index b1fcaf7209..6decec9705 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -1,6 +1,6 @@ from collections import defaultdict -from clickhouse_search.models.postgres_dicts import SexDict +from clickhouse_search.models.postgres_dicts import AffectedDict, SexDict from matchmaker.models import MatchmakerSubmission, MatchmakerResult from seqr.models import Dataset, IgvSample, RnaSample, Individual, Family, FamilyNote from seqr.utils.middleware import ErrorsWarningsException @@ -70,8 +70,10 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge updated_family_models = Family.objects.filter(id__in=updated_family_ids) _remove_pedigree_images(updated_family_models, user) - if updated_affected and not skip_gt_stats_rebuild: - trigger_rebuild_gt_stats(project, user) + if updated_affected: + AffectedDict.reload() + if not skip_gt_stats_rebuild: + trigger_rebuild_gt_stats(project, user) if updated_sex: SexDict.reload() From ee083da13202a77c2cee76584cd9ea3ad36a4745 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 15:45:01 -0400 Subject: [PATCH 083/119] log user --- clickhouse_search/backend/table_models.py | 12 ++++++------ clickhouse_search/search_tests.py | 1 + seqr/views/utils/individual_utils.py | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/clickhouse_search/backend/table_models.py b/clickhouse_search/backend/table_models.py index eaf34e4ee9..c8cb45a684 100644 --- a/clickhouse_search/backend/table_models.py +++ b/clickhouse_search/backend/table_models.py @@ -1,11 +1,11 @@ from clickhouse_backend import models from django.db import connections from django.db.models import Func -import logging from clickhouse_search.backend.fields import NamedTupleField +from seqr.utils.logging_utils import SeqrLogger -logger = logging.getLogger(__name__) +logger = SeqrLogger(__name__) MATERIALIZED_VIEW_META_FIELDS = [ @@ -44,8 +44,8 @@ class Meta: class RefreshableMaterializedView(IncrementalMaterializedView): @classmethod - def refresh(cls): - logger.info(f'Refreshing materialized view {cls._meta.db_table}') + def refresh(cls, user=None): + logger.info(f'Refreshing materialized view {cls._meta.db_table}', user) with connections['clickhouse_write'].cursor() as cursor: cursor.execute(f'SYSTEM START VIEW "{cls._meta.db_table}"') cursor.execute(f'SYSTEM REFRESH VIEW "{cls._meta.db_table}"') @@ -93,8 +93,8 @@ def dict_get_expression(cls, *expressions, field_names=None, force_tuple=False, return dict_get_func @classmethod - def reload(cls): - logger.info(f'Reloading dictionary {cls._meta.db_table}') + def reload(cls, user=None): + logger.info(f'Reloading dictionary {cls._meta.db_table}', user) with connections['clickhouse_write'].cursor() as cursor: cursor.execute(f'SYSTEM RELOAD DICTIONARY "{cls._meta.db_table}"') diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 88f80e2dc6..0efc9d8c32 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -2462,6 +2462,7 @@ def test_trigger_delete_project(self): annotations_qs = VariantsSnvIndel.objects.all().join_populations() updated_seqr_pops_by_key = dict(annotations_qs.values_list('key', 'seqrPop')) + self.maxDiff = None self.assertDictEqual(updated_seqr_pops_by_key, { 1: {'ac_wes': 2, 'ac_wgs': 2, 'hom_wes': 1, 'hom_wgs': 1, 'ac_affected': 4, 'hom_affected': 2}, 2: {'ac_wes': 1, 'ac_wgs': 1, 'hom_wes': 0, 'hom_wgs': 0, 'ac_affected': 2, 'hom_affected': 0}, diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index 6decec9705..6007ff8d6d 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -71,11 +71,11 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge updated_family_models = Family.objects.filter(id__in=updated_family_ids) _remove_pedigree_images(updated_family_models, user) if updated_affected: - AffectedDict.reload() + AffectedDict.reload(user) if not skip_gt_stats_rebuild: trigger_rebuild_gt_stats(project, user) if updated_sex: - SexDict.reload() + SexDict.reload(user) pedigree_json = None if get_update_json: From 0d36aff0559355146d5dc47140e58e5549191438 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 8 May 2026 15:59:08 -0400 Subject: [PATCH 084/119] fix unrelated test --- clickhouse_search/search_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 0efc9d8c32..aa72d2f722 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -2442,6 +2442,7 @@ class ClickhouseDeleteDataTests(ClickhouseSearchTestCase): @responses.activate def test_trigger_delete_project(self): + AffectedDict.reload() url = reverse(trigger_delete_project) self.check_data_manager_login(url) @@ -2462,7 +2463,6 @@ def test_trigger_delete_project(self): annotations_qs = VariantsSnvIndel.objects.all().join_populations() updated_seqr_pops_by_key = dict(annotations_qs.values_list('key', 'seqrPop')) - self.maxDiff = None self.assertDictEqual(updated_seqr_pops_by_key, { 1: {'ac_wes': 2, 'ac_wgs': 2, 'hom_wes': 1, 'hom_wgs': 1, 'ac_affected': 4, 'hom_affected': 2}, 2: {'ac_wes': 1, 'ac_wgs': 1, 'hom_wes': 0, 'hom_wgs': 0, 'ac_affected': 2, 'hom_affected': 0}, From 568d7a4388a5f02547ee034e9ed103b7253fc277 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 10:35:38 -0400 Subject: [PATCH 085/119] debug --- seqr/views/utils/individual_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index 6007ff8d6d..f225d28b4e 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -71,7 +71,7 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge updated_family_models = Family.objects.filter(id__in=updated_family_ids) _remove_pedigree_images(updated_family_models, user) if updated_affected: - AffectedDict.reload(user) + # AffectedDict.reload(user) if not skip_gt_stats_rebuild: trigger_rebuild_gt_stats(project, user) if updated_sex: From 7f07bba50828a662008310a027f3cdf7f54eccd8 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 10:35:48 -0400 Subject: [PATCH 086/119] undo debug --- seqr/views/utils/individual_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index f225d28b4e..6007ff8d6d 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -71,7 +71,7 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge updated_family_models = Family.objects.filter(id__in=updated_family_ids) _remove_pedigree_images(updated_family_models, user) if updated_affected: - # AffectedDict.reload(user) + AffectedDict.reload(user) if not skip_gt_stats_rebuild: trigger_rebuild_gt_stats(project, user) if updated_sex: From 52dcd2bbc009b605ed400539abd1ac93c8020684 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 11:17:42 -0400 Subject: [PATCH 087/119] test clean up --- clickhouse_search/search_tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index aa72d2f722..88f80e2dc6 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -2442,7 +2442,6 @@ class ClickhouseDeleteDataTests(ClickhouseSearchTestCase): @responses.activate def test_trigger_delete_project(self): - AffectedDict.reload() url = reverse(trigger_delete_project) self.check_data_manager_login(url) From c751e07c804af7ad82c92c2f074672e5797606b9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 14:16:46 -0400 Subject: [PATCH 088/119] debug case usage --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6e138c1ca1..8b98753216 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -99,7 +99,7 @@ jobs: export CLICKHOUSE_READER_USER=clickhouse_read_only export CLICKHOUSE_WRITER_USER=clickhouse_test export CLICKHOUSE_SERVICE_HOSTNAME=localhost - coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data seqr matchmaker panelapp clickhouse_search + coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 clickhouse_search coverage report -m --fail-under=99 coverage report | (! grep [1-8][0-9]%) coverage report | (! grep 9[0-5]%) From 913bae0149803804fcea7f105deb1f2c18e16a19 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 14:17:17 -0400 Subject: [PATCH 089/119] undo --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 8b98753216..6e138c1ca1 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -99,7 +99,7 @@ jobs: export CLICKHOUSE_READER_USER=clickhouse_read_only export CLICKHOUSE_WRITER_USER=clickhouse_test export CLICKHOUSE_SERVICE_HOSTNAME=localhost - coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 clickhouse_search + coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data seqr matchmaker panelapp clickhouse_search coverage report -m --fail-under=99 coverage report | (! grep [1-8][0-9]%) coverage report | (! grep 9[0-5]%) From cf65b17f04c1bc6dd0fa5b1d9087f3a2f0d42a86 Mon Sep 17 00:00:00 2001 From: hanars Date: Mon, 11 May 2026 14:18:08 -0400 Subject: [PATCH 090/119] debug teardown scope issue --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6e138c1ca1..8b98753216 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -99,7 +99,7 @@ jobs: export CLICKHOUSE_READER_USER=clickhouse_read_only export CLICKHOUSE_WRITER_USER=clickhouse_test export CLICKHOUSE_SERVICE_HOSTNAME=localhost - coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data seqr matchmaker panelapp clickhouse_search + coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 clickhouse_search coverage report -m --fail-under=99 coverage report | (! grep [1-8][0-9]%) coverage report | (! grep 9[0-5]%) From be30bff0ad8b01c65d07fc567e39fec0e8a2b953 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 14:26:17 -0400 Subject: [PATCH 091/119] tets logs --- clickhouse_search/search_tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 88f80e2dc6..4f5d049a74 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -2444,6 +2444,7 @@ class ClickhouseDeleteDataTests(ClickhouseSearchTestCase): def test_trigger_delete_project(self): url = reverse(trigger_delete_project) self.check_data_manager_login(url) + self.reset_logs() Project.objects.filter(guid='R0001_1kg').update(genome_version='38') body = {'project': 'R0001_1kg', 'datasetType': 'SNV_INDEL'} @@ -2460,6 +2461,8 @@ def test_trigger_delete_project(self): self.assertEqual(EntriesSnvIndel.objects.filter(project_guid='R0001_1kg').count(), 0) self.assertEqual(ProjectGtStatsSnvIndel.objects.filter(project_guid='R0001_1kg').count(), 0) + self.assert_json_logs(self.data_manager_user, [('Done', None)]) + annotations_qs = VariantsSnvIndel.objects.all().join_populations() updated_seqr_pops_by_key = dict(annotations_qs.values_list('key', 'seqrPop')) self.assertDictEqual(updated_seqr_pops_by_key, { From 5cbed9194df1b2c946186c7234fbd8993cc916d9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 15:11:50 -0400 Subject: [PATCH 092/119] debug --- clickhouse_search/search_tests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 4f5d049a74..b3f0230551 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -2461,7 +2461,10 @@ def test_trigger_delete_project(self): self.assertEqual(EntriesSnvIndel.objects.filter(project_guid='R0001_1kg').count(), 0) self.assertEqual(ProjectGtStatsSnvIndel.objects.filter(project_guid='R0001_1kg').count(), 0) - self.assert_json_logs(self.data_manager_user, [('Done', None)]) + self.assert_json_logs(self.data_manager_user, [ + ('Refreshing materialized view GRCh38/SNV_INDEL/project_gt_stats_to_gt_stats_mv', None), + ('', None) + ]) annotations_qs = VariantsSnvIndel.objects.all().join_populations() updated_seqr_pops_by_key = dict(annotations_qs.values_list('key', 'seqrPop')) From 24aec7a84b306a599a50e88e27e2780cd8d2b0ab Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 15:24:03 -0400 Subject: [PATCH 093/119] debug --- clickhouse_search/search_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index b3f0230551..f3f4acde27 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -2461,7 +2461,7 @@ def test_trigger_delete_project(self): self.assertEqual(EntriesSnvIndel.objects.filter(project_guid='R0001_1kg').count(), 0) self.assertEqual(ProjectGtStatsSnvIndel.objects.filter(project_guid='R0001_1kg').count(), 0) - self.assert_json_logs(self.data_manager_user, [ + self.assert_json_logs(None, [ ('Refreshing materialized view GRCh38/SNV_INDEL/project_gt_stats_to_gt_stats_mv', None), ('', None) ]) From 065e194df1a1a7125fa09d7dc2ee197d41c3ac95 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 15:30:33 -0400 Subject: [PATCH 094/119] debug --- clickhouse_search/search_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index f3f4acde27..d049abb00b 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -2463,6 +2463,7 @@ def test_trigger_delete_project(self): self.assert_json_logs(None, [ ('Refreshing materialized view GRCh38/SNV_INDEL/project_gt_stats_to_gt_stats_mv', None), + ('Reloading dictionary GRCh38/SNV_INDEL/gt_stats_dict', None), ('', None) ]) From 58cf829c7b27ee0ac669fa1a7501a396aac1fd3e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 15:38:38 -0400 Subject: [PATCH 095/119] clean up --- clickhouse_search/search_tests.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index d049abb00b..88f80e2dc6 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -2444,7 +2444,6 @@ class ClickhouseDeleteDataTests(ClickhouseSearchTestCase): def test_trigger_delete_project(self): url = reverse(trigger_delete_project) self.check_data_manager_login(url) - self.reset_logs() Project.objects.filter(guid='R0001_1kg').update(genome_version='38') body = {'project': 'R0001_1kg', 'datasetType': 'SNV_INDEL'} @@ -2461,12 +2460,6 @@ def test_trigger_delete_project(self): self.assertEqual(EntriesSnvIndel.objects.filter(project_guid='R0001_1kg').count(), 0) self.assertEqual(ProjectGtStatsSnvIndel.objects.filter(project_guid='R0001_1kg').count(), 0) - self.assert_json_logs(None, [ - ('Refreshing materialized view GRCh38/SNV_INDEL/project_gt_stats_to_gt_stats_mv', None), - ('Reloading dictionary GRCh38/SNV_INDEL/gt_stats_dict', None), - ('', None) - ]) - annotations_qs = VariantsSnvIndel.objects.all().join_populations() updated_seqr_pops_by_key = dict(annotations_qs.values_list('key', 'seqrPop')) self.assertDictEqual(updated_seqr_pops_by_key, { From 11d9385c195e6f3303d46c5df83587a4040b23c5 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 15:42:15 -0400 Subject: [PATCH 096/119] debug --- clickhouse_search/search_tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index 88f80e2dc6..ee23057e49 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -2445,6 +2445,11 @@ def test_trigger_delete_project(self): url = reverse(trigger_delete_project) self.check_data_manager_login(url) + annotations_qs = VariantsSnvIndel.objects.filter(key=1).join_populations() + self.assertDictEqual(dict(annotations_qs.values_list('key', 'seqrPop')), { + 1: {'ac_wes': 3, 'ac_wgs': 5, 'hom_wes': 1, 'hom_wgs': 2, 'ac_affected': 7, 'hom_affected': 3}, + }) + Project.objects.filter(guid='R0001_1kg').update(genome_version='38') body = {'project': 'R0001_1kg', 'datasetType': 'SNV_INDEL'} response = self.client.post( From 7c68c3be00b82848dd1aa2271e23b8b9af141124 Mon Sep 17 00:00:00 2001 From: hanars Date: Mon, 11 May 2026 15:52:14 -0400 Subject: [PATCH 097/119] isolate failed test --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 8b98753216..7ecf6196af 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -99,7 +99,7 @@ jobs: export CLICKHOUSE_READER_USER=clickhouse_read_only export CLICKHOUSE_WRITER_USER=clickhouse_test export CLICKHOUSE_SERVICE_HOSTNAME=localhost - coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 clickhouse_search + coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 clickhouse_search.search_tests.ClickhouseDeleteDataTests coverage report -m --fail-under=99 coverage report | (! grep [1-8][0-9]%) coverage report | (! grep 9[0-5]%) From 9de61be967e5f5a1ba0013bcdb7f716380644364 Mon Sep 17 00:00:00 2001 From: hanars Date: Mon, 11 May 2026 15:59:55 -0400 Subject: [PATCH 098/119] run all search tests --- .github/workflows/unit-tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 7ecf6196af..9ae1e38c41 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -99,8 +99,7 @@ jobs: export CLICKHOUSE_READER_USER=clickhouse_read_only export CLICKHOUSE_WRITER_USER=clickhouse_test export CLICKHOUSE_SERVICE_HOSTNAME=localhost - coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 clickhouse_search.search_tests.ClickhouseDeleteDataTests - coverage report -m --fail-under=99 + coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 clickhouse_search.search_tests coverage report | (! grep [1-8][0-9]%) coverage report | (! grep 9[0-5]%) From cb8e01e87282d015fd0fafe63dae5fc5bbb94b6d Mon Sep 17 00:00:00 2001 From: hanars Date: Mon, 11 May 2026 16:06:18 -0400 Subject: [PATCH 099/119] run more tests --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 9ae1e38c41..825fff6cdc 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -99,7 +99,7 @@ jobs: export CLICKHOUSE_READER_USER=clickhouse_read_only export CLICKHOUSE_WRITER_USER=clickhouse_test export CLICKHOUSE_SERVICE_HOSTNAME=localhost - coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 clickhouse_search.search_tests + coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 clickhouse_search coverage report | (! grep [1-8][0-9]%) coverage report | (! grep 9[0-5]%) From 5000246b684f069a8642cf6184c5db1cdb9664b7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 16:08:58 -0400 Subject: [PATCH 100/119] proper cleanup for reload clinvar tests --- .../management/tests/reload_clinvar_all_variants_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py index c22bff5763..e8a0936f3c 100644 --- a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py +++ b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py @@ -14,6 +14,7 @@ ClinvarAllVariantsGRCh37SnvIndel, ClinvarAllVariantsMito, ) from reference_data.models import DataVersions +from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin WEEKLY_XML_RELEASE_HEADER = '''''' WEEKLY_XML_RELEASE_DATA = WEEKLY_XML_RELEASE_HEADER + ''' @@ -82,7 +83,7 @@ @mock.patch('clickhouse_search.management.commands.reload_clinvar_all_variants.safe_post_to_slack') @mock.patch('clickhouse_search.management.commands.reload_clinvar_all_variants.logger.info') -class ReloadClinvarAllVariantsTest(TestCase): +class ReloadClinvarAllVariantsTest(DifferentDbTransactionSupportMixin, TestCase): databases = '__all__' fixtures = ['clinvar_all_variants'] From 53e27f1dcbaeae41f71dab4dfe3dba9fdbf3090a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 16:10:26 -0400 Subject: [PATCH 101/119] additional test cleanup --- clickhouse_search/management/tests/register_caids_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/management/tests/register_caids_tests.py b/clickhouse_search/management/tests/register_caids_tests.py index 9985799c50..463d31bdad 100644 --- a/clickhouse_search/management/tests/register_caids_tests.py +++ b/clickhouse_search/management/tests/register_caids_tests.py @@ -11,6 +11,7 @@ VariantDetailsSnvIndel, ) from reference_data.models import DataVersions +from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin MOCK_RESPONSE = [ { @@ -94,7 +95,7 @@ @mock.patch('clickhouse_search.management.commands.register_caids.logger') @mock.patch("clickhouse_search.management.commands.register_caids.safe_post_to_slack") -class RegisterCaidsEmptyDatabaseTest(TestCase): +class RegisterCaidsEmptyDatabaseTest(DifferentDbTransactionSupportMixin, TestCase): databases = "__all__" fixtures = [] From 01e70e5e7105cd9561642b3e97d37e16f156e893 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 16:14:37 -0400 Subject: [PATCH 102/119] clean up --- clickhouse_search/search_tests.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index ee23057e49..88f80e2dc6 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -2445,11 +2445,6 @@ def test_trigger_delete_project(self): url = reverse(trigger_delete_project) self.check_data_manager_login(url) - annotations_qs = VariantsSnvIndel.objects.filter(key=1).join_populations() - self.assertDictEqual(dict(annotations_qs.values_list('key', 'seqrPop')), { - 1: {'ac_wes': 3, 'ac_wgs': 5, 'hom_wes': 1, 'hom_wgs': 2, 'ac_affected': 7, 'hom_affected': 3}, - }) - Project.objects.filter(guid='R0001_1kg').update(genome_version='38') body = {'project': 'R0001_1kg', 'datasetType': 'SNV_INDEL'} response = self.client.post( From 70d8654c1f853639791850a29fec408e3d477a37 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 16:20:34 -0400 Subject: [PATCH 103/119] clean up --- clickhouse_search/management/tests/register_caids_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/management/tests/register_caids_tests.py b/clickhouse_search/management/tests/register_caids_tests.py index 463d31bdad..963633e431 100644 --- a/clickhouse_search/management/tests/register_caids_tests.py +++ b/clickhouse_search/management/tests/register_caids_tests.py @@ -114,7 +114,7 @@ def test_register_caids(self, mock_safe_post_to_slack, mock_logger): @mock.patch('clickhouse_search.management.commands.register_caids.logger') @mock.patch("clickhouse_search.management.commands.register_caids.safe_post_to_slack") -class RegisterCaidsTest(TestCase): +class RegisterCaidsTest(DifferentDbTransactionSupportMixin, TestCase): databases = "__all__" fixtures = ["variant_details_for_update"] From fda9cd3ba88ce05f39ffbdd5d2514c3ab982578c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 16:23:20 -0400 Subject: [PATCH 104/119] fix test fixtures --- .../management/tests/reload_clinvar_all_variants_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py index e8a0936f3c..e7a1ce3ad1 100644 --- a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py +++ b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py @@ -85,7 +85,7 @@ @mock.patch('clickhouse_search.management.commands.reload_clinvar_all_variants.logger.info') class ReloadClinvarAllVariantsTest(DifferentDbTransactionSupportMixin, TestCase): databases = '__all__' - fixtures = ['clinvar_all_variants'] + fixtures = ['test_conflicting_data_from_submitters'] @responses.activate def test_update_with_no_previous_version(self, mock_logger, mock_safe_post_to_slack): @@ -379,6 +379,7 @@ def test_conflicting_data_from_submitters(self, mock_logger, mock_safe_post_to_s body=gzip.compress(missing_conflicting_pathogenicities.encode()), stream=True, ) + ClinvarAllVariantsSnvIndel.objects.using('clickhouse_write').all().delete() with self.assertRaisesMessage(CommandError, 'Failed to find the conflicting pathogenicities node'): call_command('reload_clinvar_all_variants') From 70e23531ed1214e036ec630110b8642344fb6d23 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 16:37:22 -0400 Subject: [PATCH 105/119] fix copy paste --- .../management/tests/reload_clinvar_all_variants_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py index e7a1ce3ad1..4892a7221a 100644 --- a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py +++ b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py @@ -85,7 +85,7 @@ @mock.patch('clickhouse_search.management.commands.reload_clinvar_all_variants.logger.info') class ReloadClinvarAllVariantsTest(DifferentDbTransactionSupportMixin, TestCase): databases = '__all__' - fixtures = ['test_conflicting_data_from_submitters'] + fixtures = ['clinvar_all_variants'] @responses.activate def test_update_with_no_previous_version(self, mock_logger, mock_safe_post_to_slack): From 50de9d0312230747e585986a8c611d56d63cccbd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 16:46:59 -0400 Subject: [PATCH 106/119] debug remov etests --- .../management/tests/register_caids_tests.py | 256 ----------- .../reload_clinvar_all_variants_tests.py | 424 ------------------ 2 files changed, 680 deletions(-) delete mode 100644 clickhouse_search/management/tests/register_caids_tests.py delete mode 100644 clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py diff --git a/clickhouse_search/management/tests/register_caids_tests.py b/clickhouse_search/management/tests/register_caids_tests.py deleted file mode 100644 index 963633e431..0000000000 --- a/clickhouse_search/management/tests/register_caids_tests.py +++ /dev/null @@ -1,256 +0,0 @@ -import mock -import responses -from django.core.management import call_command -from django.core.management.base import CommandError -from django.test import TestCase -from settings import SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL - - -from clickhouse_search.management.commands.register_caids import ALLELE_REGISTRY_HEADERS -from clickhouse_search.models.search_models import ( - VariantDetailsSnvIndel, -) -from reference_data.models import DataVersions -from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin - -MOCK_RESPONSE = [ - { - '@id': 'http://reg.genome.network/allele/CA997563840', - 'genomicAlleles': [ - { - 'chromosome': '1', - 'coordinates': [ - { - 'allele': '', # alt allele is '' - 'end': 91511686, - 'referenceAllele': 'T', - 'start': 91511686, - }, - ], - 'referenceGenome': 'GRCh38', - }, - ], - 'externalRecords': { - 'gnomAD_4': [{'id': '1-91511686-T-G'}], - }, # has gnomad ID - }, - { - '@id': 'http://reg.genome.network/allele/CA16716503', - 'genomicAlleles': [ - { - 'chromosome': '1', - 'coordinates': [ - { - 'allele': 'C', - 'end': 10131, - 'referenceAllele': '', # ref allele is '' and does not have a gnomad ID - 'start': 10131, - }, - ], - 'referenceGenome': 'GRCh38', - }, - ], - }, - { - '@id': 'http://reg.genome.network/allele/CA997563845', - 'genomicAlleles': [ - { - 'chromosome': '1', - 'coordinates': [ - { - 'allele': 'A', - 'end': 10146, - 'referenceAllele': 'ACC', - 'start': 10146, - }, - ], - 'referenceGenome': 'GRCh38', - }, - ], - 'externalRecords': {'gnomAD_4': [{'id': '1-10146-ACC-A'}]}, - }, - { - 'description': 'Given allele cannot be mapped in consistent way to reference genome.', - 'errorType': 'InternalServerError', - 'inputLine': 'Cannot align NC_000001.10 [10468,10469).', - 'message': '1\t10469\trs370233998\tC\tG\t.\t.\t.', - }, - { - '@id': 'http://reg.genome.network/allele/CAXXX', - 'genomicAlleles': [ - { - 'chromosome': '1', - 'coordinates': [ - { - # missing allele and referenceAllele - 'end': 10131, - 'start': 10131, - }, - ], - 'referenceGenome': 'GRCh38', - }, - ], - }, -] - -@mock.patch('clickhouse_search.management.commands.register_caids.logger') -@mock.patch("clickhouse_search.management.commands.register_caids.safe_post_to_slack") -class RegisterCaidsEmptyDatabaseTest(DifferentDbTransactionSupportMixin, TestCase): - databases = "__all__" - fixtures = [] - - @responses.activate - def test_register_caids(self, mock_safe_post_to_slack, mock_logger): - call_command("register_caids", batch_size=3) - mock_logger.info.assert_called_with( - 'Attempting to register caids from key: 0' - ) - mock_logger.warning.assert_not_called() - dv_37 = DataVersions.objects.get(data_model_name='37/ClingenAlleleRegistry') - self.assertEqual(dv_37.version, '0') - dv_38 = DataVersions.objects.get(data_model_name='38/ClingenAlleleRegistry') - self.assertEqual(dv_38.version, '0') - - -@mock.patch('clickhouse_search.management.commands.register_caids.logger') -@mock.patch("clickhouse_search.management.commands.register_caids.safe_post_to_slack") -class RegisterCaidsTest(DifferentDbTransactionSupportMixin, TestCase): - databases = "__all__" - fixtures = ["variant_details_for_update"] - - @responses.activate - def test_bad_responses(self, mock_safe_post_to_slack, mock_logger): - responses.add( - responses.PUT, - "https://reg.genome.network/alleles", - match=[ - responses.matchers.query_param_matcher({ - "file": "vcf", - "fields": "none @id genomicAlleles externalRecords.gnomAD_4.id", - }, strict_match=False), - ], - status=200, - json={'errorType': 'InternalServerError'} - ) - with self.assertRaisesMessage(CommandError, 'Failed in 38/ClingenAlleleRegistry curr_key: 3'): - call_command("register_caids", batch_size=5) - mock_safe_post_to_slack.assert_not_called() - - responses.reset() - mock_safe_post_to_slack.reset_mock() - mock_logger.reset_mock() - - responses.add( - responses.PUT, - "https://reg.genome.network/alleles", - match=[ - responses.matchers.query_param_matcher({ - "file": "vcf", - "fields": "none @id genomicAlleles externalRecords.gnomAD_4.id", - }, strict_match=False), - ], - status=200, - json={'non': 'list'} - ) - with self.assertRaisesMessage(CommandError, 'Failed in 38/ClingenAlleleRegistry curr_key: 3'): - call_command("register_caids", batch_size=5) - mock_safe_post_to_slack.assert_not_called() - mock_logger.exception.assert_called_with( - 'Failed in 38/ClingenAlleleRegistry curr_key: 3' - ) - - @responses.activate - def test_failure(self, mock_safe_post_to_slack, mock_logger): - responses.add( - responses.PUT, - "https://reg.genome.network/alleles", - match=[ - responses.matchers.query_param_matcher({ - "file": "vcf", - "fields": "none @id genomicAlleles externalRecords.gnomAD_4.id", - }, strict_match=False), - ], - status=500, - ) - with self.assertRaisesMessage(CommandError, 'Failed in 38/ClingenAlleleRegistry curr_key: 3'): - call_command("register_caids", batch_size=5) - mock_safe_post_to_slack.assert_not_called() - mock_logger.exception.assert_called_with( - 'Failed in 38/ClingenAlleleRegistry curr_key: 3' - ) - dv = DataVersions.objects.get(data_model_name='38/ClingenAlleleRegistry') - self.assertEqual(dv.version, '3') - - @responses.activate - def test_register_caids(self, mock_safe_post_to_slack, mock_logger): - responses.add( - responses.PUT, - "https://reg.genome.network/alleles", - match=[ - responses.matchers.query_param_matcher({ - "file": "vcf", - "fields": "none @id genomicAlleles externalRecords.gnomAD_4.id", - }, strict_match=False), - ], - status=200, - json=MOCK_RESPONSE - ) - call_command("register_caids", batch_size=3) - self.assertEqual(len(responses.calls), 2) - self.assertEqual( - [call.request.body for call in responses.calls], - [ - "\n".join( - [ - *ALLELE_REGISTRY_HEADERS['38'], - '1\t91511686\t.\tT\tG\t.\t.\t.', - '1\t10146\t.\tACC\tA\t.\t.\t.', - '1\t94818\t.\tT\tC\t.\t.\t.', - ] - ) + "\n", - "\n".join( - [ - *ALLELE_REGISTRY_HEADERS['38'], - '7\t143270172\t.\tA\tG\t.\t.\t.', - '7\t9310123\t.\tT\tC\t.\t.\t.', - ] - ) + "\n", - ], - ) - mock_safe_post_to_slack.assert_has_calls([ - mock.call(SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, "Registered Clingen Allele IDs for 7 recently loaded GRCh38 variants"), - ]) - dv_37 = DataVersions.objects.get(data_model_name='37/ClingenAlleleRegistry') - self.assertEqual(dv_37.version, '11') - dv_38 = DataVersions.objects.get(data_model_name='38/ClingenAlleleRegistry') - self.assertEqual(dv_38.version, '10') - mock_logger.info.assert_called_with( - "2 registered variant(s) cannot be mapped back to ours. \n" - "First unmappable variant:\n{'@id': 'http://reg.genome.network/allele/CA16716503', 'genomicAlleles': [{'chromosome': '1', 'coordinates': [{'allele': 'C', 'end': 10131, 'referenceAllele': '', 'start': 10131}], 'referenceGenome': 'GRCh38'}]}" - ) - mock_logger.warning.assert_called_with( - '1 failed. First error: \n' - 'TYPE: InternalServerError\n' - 'DESCRIPTION: Given allele cannot be mapped in consistent way to reference genome.\n' - 'MESSAGE: 1\t10469\trs370233998\tC\tG\t.\t.\t.\n' - 'INPUT_LINE: Cannot align NC_000001.10 [10468,10469).', - ) - vd = VariantDetailsSnvIndel.objects.get(variant_id='1-91511686-T-G') - self.assertEqual(vd.caid, 'CA997563840') - - # Ensure re-calling is a no-op - responses.reset() - mock_safe_post_to_slack.reset_mock() - mock_logger.reset_mock() - call_command("register_caids", batch_size=3) - mock_logger.info.assert_called_with( - 'Attempting to register caids from key: 10', - ) - mock_logger.warning.assert_not_called() - mock_safe_post_to_slack.assert_not_called() - self.assertEqual(len(responses.calls), 0) - dv_37 = DataVersions.objects.get(data_model_name='37/ClingenAlleleRegistry') - self.assertEqual(dv_37.version, '11') - dv_38 = DataVersions.objects.get(data_model_name='38/ClingenAlleleRegistry') - self.assertEqual(dv_38.version, '10') - diff --git a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py deleted file mode 100644 index 4892a7221a..0000000000 --- a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py +++ /dev/null @@ -1,424 +0,0 @@ -import datetime -import gzip -import mock -import responses -from django.core.management import call_command -from django.core.management.base import CommandError -from django.forms.models import model_to_dict -from django.test import TestCase -from settings import SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL - -from clickhouse_search.management.commands.reload_clinvar_all_variants import BATCH_SIZE, WEEKLY_XML_RELEASE -from clickhouse_search.models.reference_data_models import ( - ClinvarSnvIndel, ClinvarGRCh37SnvIndel, ClinvarAllVariantsSnvIndel, - ClinvarAllVariantsGRCh37SnvIndel, ClinvarAllVariantsMito, -) -from reference_data.models import DataVersions -from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin - -WEEKLY_XML_RELEASE_HEADER = '''''' -WEEKLY_XML_RELEASE_DATA = WEEKLY_XML_RELEASE_HEADER + ''' - - - - - - - - - - - criteria provided, conflicting classifications - Conflicting classifications of pathogenicity; association; risk factor - Pathogenic(18); Likely pathogenic(9); Pathogenic, low penetrance(1); Established risk allele(1); Likely risk allele(1); Uncertain significance(1); VUS-high(1) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Pathogenic/Likely pathogenic/Pathogenic, low penetrance/Established risk allele - - - - - -''' - - -@mock.patch('clickhouse_search.management.commands.reload_clinvar_all_variants.safe_post_to_slack') -@mock.patch('clickhouse_search.management.commands.reload_clinvar_all_variants.logger.info') -class ReloadClinvarAllVariantsTest(DifferentDbTransactionSupportMixin, TestCase): - databases = '__all__' - fixtures = ['clinvar_all_variants'] - - @responses.activate - def test_update_with_no_previous_version(self, mock_logger, mock_safe_post_to_slack): - DataVersions.objects.all().delete() - ClinvarAllVariantsSnvIndel.objects.using('clickhouse_write').all().delete() - responses.add(responses.GET, WEEKLY_XML_RELEASE, status=200, body=gzip.compress(WEEKLY_XML_RELEASE_DATA.encode()), stream=True) - call_command('reload_clinvar_all_variants') - mock_logger.assert_called_with('Updating Clinvar ClickHouse tables to 2025-06-30 from None.') - self.assertEqual(ClinvarAllVariantsSnvIndel.objects.count(), 1) - - @responses.activate - def test_new_version_already_exists(self, mock_logger, mock_safe_post_to_slack): - version_obj = DataVersions.objects.filter(data_model_name='Clinvar').first() - version_obj.version = '2025-06-30' - version_obj.save() - responses.add(responses.GET, WEEKLY_XML_RELEASE, status=200, body=gzip.compress(WEEKLY_XML_RELEASE_DATA.encode()), stream=True) - call_command('reload_clinvar_all_variants') - mock_logger.assert_called_with('Clinvar ClickHouse tables already successfully updated to 2025-06-30, gracefully exiting.') - - @responses.activate - def test_parse_variants_all_types(self, mock_logger, mock_safe_post_to_slack): - responses.add(responses.GET, WEEKLY_XML_RELEASE, status=200, body=gzip.compress(WEEKLY_XML_RELEASE_DATA.encode()), stream=True) - call_command('reload_clinvar_all_variants') - mock_logger.assert_called_with('Updating Clinvar ClickHouse tables to 2025-06-30 from 2025-06-23.') - seqr_clinvar_snv_indel_models = ClinvarSnvIndel.objects.all() - self.assertEqual(seqr_clinvar_snv_indel_models.count(), 1) - self.assertDictEqual( - model_to_dict(seqr_clinvar_snv_indel_models.first()), - { - 'allele_id': 20642, - 'assertions': ['association', 'risk_factor'], - 'conditions': [ - 'Breast carcinoma', - 'CHEK2-related cancer predisposition', - 'CHEK2-related disorder', - 'Hereditary cancer-predisposing syndrome', - ], - 'conflicting_pathogenicities': [ - {'pathogenicity': 'Pathogenic', 'count': 19}, - {'pathogenicity': 'Likely_pathogenic', 'count': 9}, - {'pathogenicity': 'Uncertain_significance', 'count': 2}, - {'pathogenicity': 'Established_risk_allele', 'count': 1}, - {'pathogenicity': 'Likely_risk_allele', 'count': 1}, - ], - 'gold_stars': 1, - 'key': 12, - 'pathogenicity': 'Conflicting_classifications_of_pathogenicity', - 'submitters': [ - 'Fulgent Genetics, Fulgent Genetics', - 'Revvity Omics, Revvity', - 'University of Washington Department of Laboratory Medicine, University of Washington' - ] - }, - ) - - seqr_clinvar_grch37_snv_indel_models = ClinvarGRCh37SnvIndel.objects.all() - self.assertEqual(seqr_clinvar_grch37_snv_indel_models.count(), 0) - clinvar_all_variants_grch37_snv_indel_models = ClinvarAllVariantsGRCh37SnvIndel.objects.all() - self.assertEqual(clinvar_all_variants_grch37_snv_indel_models.count(), 1) - clinvar_all_variants_mito_models = ClinvarAllVariantsMito.objects.all() - self.assertDictEqual( - model_to_dict(clinvar_all_variants_mito_models.first()), - { - 'allele_id': 20642, - 'assertions': ['low_penetrance'], - 'conditions': [], - 'conflicting_pathogenicities': None, - 'gold_stars': None, - 'pathogenicity': 'Pathogenic/Likely_pathogenic/Established_risk_allele', - 'submitters': [], - 'variant_id': 'M-123-G-A', - 'version': datetime.date(2025, 6, 30) - } - ) - - # Version in Postgres. - dv = DataVersions.objects.get(data_model_name='Clinvar') - self.assertEqual(dv.version, '2025-06-30') - mock_safe_post_to_slack.assert_called_with( - SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, - 'Successfully updated Clinvar ClickHouse tables to 2025-06-30.', - ) - - - @responses.activate - def test_batching(self, mock_logger, mock_safe_post_to_slack): - # Dynamically build many variants - data = WEEKLY_XML_RELEASE_HEADER - for i in range(BATCH_SIZE * 2 + 10): - data += f''' - - - - - - - - - - - - - - - ''' - data += '' - responses.add(responses.GET, WEEKLY_XML_RELEASE, status=200, body=gzip.compress(data.encode()), stream=True) - call_command('reload_clinvar_all_variants') - mock_logger.assert_called_with('Updating Clinvar ClickHouse tables to 2025-06-30 from 2025-06-23.') - self.assertEqual(ClinvarAllVariantsSnvIndel.objects.all().count(), BATCH_SIZE * 2 + 10) - self.assertEqual(ClinvarAllVariantsSnvIndel.objects.first().pathogenicity, ClinvarAllVariantsSnvIndel.CLINVAR_DEFAULT_PATHOGENICITY) - self.assertIsNone(ClinvarAllVariantsSnvIndel.objects.first().gold_stars) - - @responses.activate - def test_malformed_variants(self, mock_logger, mock_safe_post_to_slack): - # Test errors that should still raise (not unenumerated values) - for description, review_status, conflicting_pathogenicities, error_message in [ - ("Pathogenic", "unhandled", None, 'Found unexpected review status unhandled'), - ("Conflicting classifications of pathogenicity", None, "Pathogenic;", 'Failed to correctly parse conflicting pathogenicity counts: Pathogenic;'), - ("Conflicting classifications of pathogenicity", None, None, 'Failed to find the conflicting pathogenicities node'), - ]: - data = f''' - - - - - - - - - - - {description} - {f"{review_status}" if review_status else ""} - {f"{conflicting_pathogenicities}" if conflicting_pathogenicities else ""} - - - - - ''' - responses.add( - responses.GET, - WEEKLY_XML_RELEASE, - status=200, - body=gzip.compress(data.encode()), - stream=True, - ) - with self.assertRaisesMessage(CommandError, error_message): - call_command('reload_clinvar_all_variants') - - # Test that unenumerated values are now collected instead of raising - unenumerated_test_cases = [ - { - 'allele_id': 1, - 'description': 'Pathogenic-ey', - 'explanation': None, - 'alert_type': 'Assertion', - 'alert_value': 'Pathogenic-ey', - }, - { - 'allele_id': 2, - 'description': 'Pathogenic; but unknown assertion', - 'explanation': None, - 'alert_type': 'Assertion', - 'alert_value': 'but unknown assertion', - }, - { - 'allele_id': 3, - 'description': 'Conflicting classifications of pathogenicity', - 'explanation': 'Pathogenic(18); unhandled(1)', - 'alert_type': 'Conflicting Pathogenicity', - 'alert_value': 'unhandled', - }, - ] - data = WEEKLY_XML_RELEASE_HEADER - for case in unenumerated_test_cases: - data += f''' - - - - - - - - - - {case['description']} - {f"{case['explanation']}" if case['explanation'] else ""} - - - - - ''' - data += '' - responses.add( - responses.GET, - WEEKLY_XML_RELEASE, - status=200, - body=gzip.compress(data.encode()), - stream=True, - ) - DataVersions.objects.all().delete() - ClinvarAllVariantsSnvIndel.objects.using('clickhouse_write').all().delete() - call_command('reload_clinvar_all_variants') - # Command should succeed and collect all unenumerated values in single slack message - slack_message = mock_safe_post_to_slack.call_args[0][1] - self.assertIn(f'Found {len(unenumerated_test_cases)} unenumerated value(s) during parsing:', slack_message) - for case in unenumerated_test_cases: - self.assertIn(f"- {case['alert_type']}: '{case['alert_value']}' (Allele ID: {case['allele_id']})", slack_message) - - # Variants with missing/equivalent alleles and positions are skipped - for simple_allele_attrs, sequence_location_attrs in [ - # Case 1: Missing AlleleId in - ("", 'Assembly="GRCh38" Chr="1" positionVCF="1" referenceAlleleVCF="G" alternateAlleleVCF="A"'), - - # Case 2: Missing alternateAlleleVCF in - ('AlleleID="5603"', 'Assembly="GRCh38" Chr="1" positionVCF="1" referenceAlleleVCF="G"'), - - # Case 3: ref/alt are equal - ('AlleleID="5603"', 'Assembly="GRCh38" Chr="1" positionVCF="1" referenceAlleleVCF="G" alternateAlleleVCF="G"'), - ]: - data = f''' - - - - - - - - - - - - - - - - - - Pathogenic - criteria provided, conflicting classifications - - - - - ''' - responses.add( - responses.GET, - WEEKLY_XML_RELEASE, - status=200, - body=gzip.compress(data.encode()), - stream=True, - ) - DataVersions.objects.all().delete() - ClinvarAllVariantsSnvIndel.objects.using('clickhouse_write').all().delete() - call_command('reload_clinvar_all_variants') - self.assertEqual(ClinvarAllVariantsSnvIndel.objects.count(), 0) - self.assertEqual(ClinvarAllVariantsGRCh37SnvIndel.objects.count(), 0) - mock_safe_post_to_slack.assert_called_with( - SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, - 'Successfully updated Clinvar ClickHouse tables to 2025-06-30.', - ) - - @responses.activate - def test_conflicting_data_from_submitters(self, mock_logger, mock_safe_post_to_slack): - missing_conflicting_pathogenicities = WEEKLY_XML_RELEASE_HEADER + ''' - - - - - - - - - - conflicting data from submitters - - - - - - ''' - responses.add( - responses.GET, - WEEKLY_XML_RELEASE, - status=200, - body=gzip.compress(missing_conflicting_pathogenicities.encode()), - stream=True, - ) - ClinvarAllVariantsSnvIndel.objects.using('clickhouse_write').all().delete() - with self.assertRaisesMessage(CommandError, 'Failed to find the conflicting pathogenicities node'): - call_command('reload_clinvar_all_variants') - - conflicting_pathogenicities = WEEKLY_XML_RELEASE_HEADER + ''' - - - - - - - - - - conflicting data from submitters - - - - - - Uncertain significance(1), Likely benign (1) - - - - - - - - ''' - responses.add( - responses.GET, - WEEKLY_XML_RELEASE, - status=200, - body=gzip.compress(conflicting_pathogenicities.encode()), - stream=True, - ) - call_command('reload_clinvar_all_variants') - self.assertEqual(ClinvarAllVariantsSnvIndel.objects.count(), 1) - self.assertEqual(ClinvarAllVariantsSnvIndel.objects.first().pathogenicity, ClinvarAllVariantsSnvIndel.CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY) - mock_safe_post_to_slack.assert_called_with( - SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, - 'Successfully updated Clinvar ClickHouse tables to 2025-06-30.', - ) From ee35c7a95197d1bb6f7146b40e9631e902dcc27d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 16:51:50 -0400 Subject: [PATCH 107/119] debug remove tests --- .../tests/set_saved_variant_key_tests.py | 133 ------------------ 1 file changed, 133 deletions(-) delete mode 100644 clickhouse_search/management/tests/set_saved_variant_key_tests.py diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py deleted file mode 100644 index 9b3d07f479..0000000000 --- a/clickhouse_search/management/tests/set_saved_variant_key_tests.py +++ /dev/null @@ -1,133 +0,0 @@ -from django.core.management import call_command -import mock - -from seqr.models import Project, Dataset, SavedVariant -from seqr.views.utils.test_utils import AnvilAuthenticationTestCase - -MOCK_GCNV_DATA = [ - b'chr\tstart\tend\tname\tsample\tsample_fix\tsvtype\tGT\tCN\tNP\tQA\tQS\tQSE\tQSS\tploidy\tstrand\tvariant_name\tID\trmsstd\tdefragmented\tvaf\tvac\tlt100_raw_calls\tlt10_highQS_rare_calls\tPASS_SAMPLE\tPASS_FREQ\tPASS_QS\tHIGH_QUALITY\tgenes_any_overlap\tgenes_any_overlap_exonsPerGene\tgenes_any_overlap_totalExons\tgenes_strict_overlap\tgenes_strict_overlap_exonsPerGene\tgenes_strict_overlap_totalExons\tgenes_CG\tgenes_LOF\tgenes_any_overlap_Ensemble_ID\tidentical\tpartial_ovl\tany_ovl\tno_ovl\tsource', - b'chr10\t122599967\t122600191\tCASE_7_Y_cnv_57308\tC1933_010-001_v1_Exome_GCP\t010-001_v1_Exome_C1933\tDEL\t1\t1\t1\t7\t7\t7\t72\t\t-\tprefix_70191\t326536\t0\tFALSE\t0.05066805\t1077\tTRUE\tTRUE\tTRUE\tFALSE\tFALSE\tFALSE\tDMBT1\t1\t1\tNA\t0\t0\tNA\tDMBT1\tENSG00000187908.19\tcluster_10_CASE_cnv_46117\tcluster_10_CASE_cnv_46117\tcluster_10_CASE_cnv_46117;prefix_19107\tFALSE\tround2', -] - - -class SetSavedVariantKeyTest(AnvilAuthenticationTestCase): - fixtures = ['users', '1kg_project', 'report_variants', 'clickhouse_saved_variants'] - - MOCK_GCNV_DATA = MOCK_GCNV_DATA - - @classmethod - def setUpTestData(cls): - Project.objects.filter(id=3).update(genome_version='38') - dataset = Dataset.objects.get(guid='S000154_na20889') - dataset.dataset_type = 'SV' - dataset.save() - dataset.active_individuals.set({17}) - for sv in SavedVariant.objects.filter(key__isnull=False): - sv.saved_variant_json = { - 'genotypes': sv.genotypes, 'populations': {'gnomad': {'af': 0.01}}, - } - if sv.guid == 'SV0000009_25000014783_r0004_no': - sv.saved_variant_json['populations']['seqr'] = {'af': 0.019480518996715546, 'ac': 3, 'an': 154} - sv.save() - SavedVariant.objects.update(key=None) - SavedVariant.objects.filter(guid='SV0000007_prefix_19107_DEL_r00').update(variant_id='prefix_19107_DEL') - - @mock.patch('clickhouse_search.management.commands.set_saved_variant_key.BATCH_SIZE', 2) - @mock.patch('seqr.utils.file_utils.subprocess.Popen') - def test_command(self, mock_subprocess): - mock_subprocess.return_value.stdout = self.MOCK_GCNV_DATA - mock_subprocess.return_value.wait.return_value = 0 - - call_command('set_saved_variant_key') - self.assert_json_logs(user=None, expected=[ - ('Finding keys for 1 MITO (GRCh38) variant ids', None), - ('Found 1 keys', None), - ('Updated batch of 1', None), - ('Updated keys for 1 MITO (GRCh38) variants', None), - ('Finding keys for 1 SNV_INDEL (GRCh38) variant ids', None), - ('Found 1 keys', None), - ('Updated batch of 2', None), - ('Updated keys for 2 SNV_INDEL (GRCh38) variants', None), - ('Finding keys for 2 SV_WGS (GRCh38) variant ids', None), - ('Found 1 keys', None), - ('Updated batch of 1', None), - ('Updated keys for 1 SV_WGS (GRCh38) variants', None), - ('No key found for 1 variants', None), - ('Finding keys for 1 SV_WES (GRCh38) variant ids', None), - ('Found 1 keys', None), - ('Updated batch of 1', None), - ('Updated keys for 1 SV_WES (GRCh38) variants', None), - ('Finding keys for 7 SNV_INDEL (GRCh37) variant ids', None), - ('Found 1 keys', None), - ('Updated batch of 1', None), - ('Updated keys for 1 SNV_INDEL (GRCh37) variants', None), - ('No key found for 6 variants', None), - ('6 variants have no key, 0 of which have no search data, 6 of which are absent from the hail backend.', None), - ('Cleared saved json for 6 variants with keys', None), - ('Done', None), - ]) - - saved_variants = list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id', 'saved_variant_json')) - expected_saved_variants = [ - {'guid': 'SV0000001_2103343353_r0390_100', 'key': None, 'variant_id': '21-3343353-GAGA-G', 'saved_variant_json': mock.ANY}, - {'guid': 'SV0000002_1248367227_r0390_100', 'key': 100, 'variant_id': '1-248367227-TC-T', 'saved_variant_json': {}}, - {'guid': 'SV0000006_1248367227_r0003_tes', 'key': 100, 'variant_id': '1-248367227-TC-T', 'saved_variant_json': {}}, - {'guid': 'SV0000006_1248367227_r0004_non', 'key': 100, 'variant_id': '1-248367227-TC-T', 'saved_variant_json': {}}, - {'guid': 'SV0000007_prefix_19107_DEL_r00', 'key': 111, 'variant_id': 'prefix_19107_DEL', 'saved_variant_json': {}}, - {'guid': 'SV0000009_25000014783_r0004_no', 'key': 100, 'variant_id': 'M-14783-T-C', 'saved_variant_json': {}}, - {'guid': 'SV0000013_prefix_19107_DEL_r00', 'key': 101, 'variant_id': 'suffix_19107_DEL_013746', 'saved_variant_json': {}}, - {'guid': 'SV0027166_191912634_r0384_rare', 'key': None, 'variant_id': '19-1912634-C-T', 'saved_variant_json': mock.ANY}, - {'guid': 'SV0027167_191912633_r0384_rare', 'key': None, 'variant_id': '19-1912633-G-T', 'saved_variant_json': mock.ANY}, - {'guid': 'SV0027168_191912632_r0384_rare', 'key': None, 'variant_id': '19-1912632-G-C', 'saved_variant_json': mock.ANY}, - {'guid': 'SV0059956_11560662_f019313_1', 'key': None, 'variant_id': '1-46859832-G-A', 'saved_variant_json': mock.ANY}, - {'guid': 'SV0059957_11562437_f019313_1', 'key': None, 'variant_id': '1-1562437-G-CA', 'saved_variant_json': mock.ANY}, - ] - self.assertListEqual(saved_variants, expected_saved_variants) - - # Reloading is a no-op - self.reset_logs() - call_command('set_saved_variant_key') - self.assert_json_logs(user=None, expected=[ - ('Finding keys for 6 SNV_INDEL (GRCh37) variant ids', None), - ('Found 0 keys', None), - ('6 variants have no key, 0 of which have no search data, 6 of which are absent from the hail backend.', None), - ('Cleared saved json for 0 variants with keys', None), - ('Done', None), - ]) - self.assertListEqual( - list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id', 'saved_variant_json')), - expected_saved_variants, - ) - -class SetSavedVariantKeyFailedMappingTest(SetSavedVariantKeyTest): - fixtures = ['users', '1kg_project', 'report_variants'] - - MOCK_GCNV_DATA = MOCK_GCNV_DATA[:1] - - @mock.patch('seqr.utils.file_utils.subprocess.Popen') - def test_command(self, mock_subprocess): - mock_subprocess.return_value.stdout = self.MOCK_GCNV_DATA - mock_subprocess.return_value.wait.return_value = 0 - - call_command('set_saved_variant_key') - self.assert_json_logs(user=None, expected=[ - ('Finding keys for 1 MITO (GRCh38) variant ids', None), - ('Found 0 keys', None), - ('Finding keys for 2 SNV_INDEL (GRCh38) variant ids', None), - ('Found 0 keys', None), - ('3 variants have no key, 1 of which have no search data, 1 of which are absent from the hail backend.', None), - ('1 remaining variants: M-14783-T-C - fam14', None), - ('Finding keys for 2 SV_WGS (GRCh38) variant ids', None), - ('Found 0 keys', None), - ('Finding keys for 2 SV_WES (GRCh38) variant ids', None), - ('Found 0 keys', None), - ('2 SV variants have no key, 0 of which have no search data', None), - ('1 remaining SV WES variants prefix_19107_DEL - 12', None), - ('1 remaining SV WGS variants suffix_19107_DEL_013746 - fam14', None), - ('Finding keys for 7 SNV_INDEL (GRCh37) variant ids', None), - ('Found 0 keys', None), - ('7 variants have no key, 0 of which have no search data, 7 of which are absent from the hail backend.', None), - ('Cleared saved json for 0 variants with keys', None), - ('Done', None), - ]) - From 34e1cd5f7093f09f0bf7e09359388c77758e315a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 16:56:36 -0400 Subject: [PATCH 108/119] add back tests --- .../management/tests/register_caids_tests.py | 256 +++++++++++ .../reload_clinvar_all_variants_tests.py | 424 ++++++++++++++++++ 2 files changed, 680 insertions(+) create mode 100644 clickhouse_search/management/tests/register_caids_tests.py create mode 100644 clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py diff --git a/clickhouse_search/management/tests/register_caids_tests.py b/clickhouse_search/management/tests/register_caids_tests.py new file mode 100644 index 0000000000..963633e431 --- /dev/null +++ b/clickhouse_search/management/tests/register_caids_tests.py @@ -0,0 +1,256 @@ +import mock +import responses +from django.core.management import call_command +from django.core.management.base import CommandError +from django.test import TestCase +from settings import SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL + + +from clickhouse_search.management.commands.register_caids import ALLELE_REGISTRY_HEADERS +from clickhouse_search.models.search_models import ( + VariantDetailsSnvIndel, +) +from reference_data.models import DataVersions +from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin + +MOCK_RESPONSE = [ + { + '@id': 'http://reg.genome.network/allele/CA997563840', + 'genomicAlleles': [ + { + 'chromosome': '1', + 'coordinates': [ + { + 'allele': '', # alt allele is '' + 'end': 91511686, + 'referenceAllele': 'T', + 'start': 91511686, + }, + ], + 'referenceGenome': 'GRCh38', + }, + ], + 'externalRecords': { + 'gnomAD_4': [{'id': '1-91511686-T-G'}], + }, # has gnomad ID + }, + { + '@id': 'http://reg.genome.network/allele/CA16716503', + 'genomicAlleles': [ + { + 'chromosome': '1', + 'coordinates': [ + { + 'allele': 'C', + 'end': 10131, + 'referenceAllele': '', # ref allele is '' and does not have a gnomad ID + 'start': 10131, + }, + ], + 'referenceGenome': 'GRCh38', + }, + ], + }, + { + '@id': 'http://reg.genome.network/allele/CA997563845', + 'genomicAlleles': [ + { + 'chromosome': '1', + 'coordinates': [ + { + 'allele': 'A', + 'end': 10146, + 'referenceAllele': 'ACC', + 'start': 10146, + }, + ], + 'referenceGenome': 'GRCh38', + }, + ], + 'externalRecords': {'gnomAD_4': [{'id': '1-10146-ACC-A'}]}, + }, + { + 'description': 'Given allele cannot be mapped in consistent way to reference genome.', + 'errorType': 'InternalServerError', + 'inputLine': 'Cannot align NC_000001.10 [10468,10469).', + 'message': '1\t10469\trs370233998\tC\tG\t.\t.\t.', + }, + { + '@id': 'http://reg.genome.network/allele/CAXXX', + 'genomicAlleles': [ + { + 'chromosome': '1', + 'coordinates': [ + { + # missing allele and referenceAllele + 'end': 10131, + 'start': 10131, + }, + ], + 'referenceGenome': 'GRCh38', + }, + ], + }, +] + +@mock.patch('clickhouse_search.management.commands.register_caids.logger') +@mock.patch("clickhouse_search.management.commands.register_caids.safe_post_to_slack") +class RegisterCaidsEmptyDatabaseTest(DifferentDbTransactionSupportMixin, TestCase): + databases = "__all__" + fixtures = [] + + @responses.activate + def test_register_caids(self, mock_safe_post_to_slack, mock_logger): + call_command("register_caids", batch_size=3) + mock_logger.info.assert_called_with( + 'Attempting to register caids from key: 0' + ) + mock_logger.warning.assert_not_called() + dv_37 = DataVersions.objects.get(data_model_name='37/ClingenAlleleRegistry') + self.assertEqual(dv_37.version, '0') + dv_38 = DataVersions.objects.get(data_model_name='38/ClingenAlleleRegistry') + self.assertEqual(dv_38.version, '0') + + +@mock.patch('clickhouse_search.management.commands.register_caids.logger') +@mock.patch("clickhouse_search.management.commands.register_caids.safe_post_to_slack") +class RegisterCaidsTest(DifferentDbTransactionSupportMixin, TestCase): + databases = "__all__" + fixtures = ["variant_details_for_update"] + + @responses.activate + def test_bad_responses(self, mock_safe_post_to_slack, mock_logger): + responses.add( + responses.PUT, + "https://reg.genome.network/alleles", + match=[ + responses.matchers.query_param_matcher({ + "file": "vcf", + "fields": "none @id genomicAlleles externalRecords.gnomAD_4.id", + }, strict_match=False), + ], + status=200, + json={'errorType': 'InternalServerError'} + ) + with self.assertRaisesMessage(CommandError, 'Failed in 38/ClingenAlleleRegistry curr_key: 3'): + call_command("register_caids", batch_size=5) + mock_safe_post_to_slack.assert_not_called() + + responses.reset() + mock_safe_post_to_slack.reset_mock() + mock_logger.reset_mock() + + responses.add( + responses.PUT, + "https://reg.genome.network/alleles", + match=[ + responses.matchers.query_param_matcher({ + "file": "vcf", + "fields": "none @id genomicAlleles externalRecords.gnomAD_4.id", + }, strict_match=False), + ], + status=200, + json={'non': 'list'} + ) + with self.assertRaisesMessage(CommandError, 'Failed in 38/ClingenAlleleRegistry curr_key: 3'): + call_command("register_caids", batch_size=5) + mock_safe_post_to_slack.assert_not_called() + mock_logger.exception.assert_called_with( + 'Failed in 38/ClingenAlleleRegistry curr_key: 3' + ) + + @responses.activate + def test_failure(self, mock_safe_post_to_slack, mock_logger): + responses.add( + responses.PUT, + "https://reg.genome.network/alleles", + match=[ + responses.matchers.query_param_matcher({ + "file": "vcf", + "fields": "none @id genomicAlleles externalRecords.gnomAD_4.id", + }, strict_match=False), + ], + status=500, + ) + with self.assertRaisesMessage(CommandError, 'Failed in 38/ClingenAlleleRegistry curr_key: 3'): + call_command("register_caids", batch_size=5) + mock_safe_post_to_slack.assert_not_called() + mock_logger.exception.assert_called_with( + 'Failed in 38/ClingenAlleleRegistry curr_key: 3' + ) + dv = DataVersions.objects.get(data_model_name='38/ClingenAlleleRegistry') + self.assertEqual(dv.version, '3') + + @responses.activate + def test_register_caids(self, mock_safe_post_to_slack, mock_logger): + responses.add( + responses.PUT, + "https://reg.genome.network/alleles", + match=[ + responses.matchers.query_param_matcher({ + "file": "vcf", + "fields": "none @id genomicAlleles externalRecords.gnomAD_4.id", + }, strict_match=False), + ], + status=200, + json=MOCK_RESPONSE + ) + call_command("register_caids", batch_size=3) + self.assertEqual(len(responses.calls), 2) + self.assertEqual( + [call.request.body for call in responses.calls], + [ + "\n".join( + [ + *ALLELE_REGISTRY_HEADERS['38'], + '1\t91511686\t.\tT\tG\t.\t.\t.', + '1\t10146\t.\tACC\tA\t.\t.\t.', + '1\t94818\t.\tT\tC\t.\t.\t.', + ] + ) + "\n", + "\n".join( + [ + *ALLELE_REGISTRY_HEADERS['38'], + '7\t143270172\t.\tA\tG\t.\t.\t.', + '7\t9310123\t.\tT\tC\t.\t.\t.', + ] + ) + "\n", + ], + ) + mock_safe_post_to_slack.assert_has_calls([ + mock.call(SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, "Registered Clingen Allele IDs for 7 recently loaded GRCh38 variants"), + ]) + dv_37 = DataVersions.objects.get(data_model_name='37/ClingenAlleleRegistry') + self.assertEqual(dv_37.version, '11') + dv_38 = DataVersions.objects.get(data_model_name='38/ClingenAlleleRegistry') + self.assertEqual(dv_38.version, '10') + mock_logger.info.assert_called_with( + "2 registered variant(s) cannot be mapped back to ours. \n" + "First unmappable variant:\n{'@id': 'http://reg.genome.network/allele/CA16716503', 'genomicAlleles': [{'chromosome': '1', 'coordinates': [{'allele': 'C', 'end': 10131, 'referenceAllele': '', 'start': 10131}], 'referenceGenome': 'GRCh38'}]}" + ) + mock_logger.warning.assert_called_with( + '1 failed. First error: \n' + 'TYPE: InternalServerError\n' + 'DESCRIPTION: Given allele cannot be mapped in consistent way to reference genome.\n' + 'MESSAGE: 1\t10469\trs370233998\tC\tG\t.\t.\t.\n' + 'INPUT_LINE: Cannot align NC_000001.10 [10468,10469).', + ) + vd = VariantDetailsSnvIndel.objects.get(variant_id='1-91511686-T-G') + self.assertEqual(vd.caid, 'CA997563840') + + # Ensure re-calling is a no-op + responses.reset() + mock_safe_post_to_slack.reset_mock() + mock_logger.reset_mock() + call_command("register_caids", batch_size=3) + mock_logger.info.assert_called_with( + 'Attempting to register caids from key: 10', + ) + mock_logger.warning.assert_not_called() + mock_safe_post_to_slack.assert_not_called() + self.assertEqual(len(responses.calls), 0) + dv_37 = DataVersions.objects.get(data_model_name='37/ClingenAlleleRegistry') + self.assertEqual(dv_37.version, '11') + dv_38 = DataVersions.objects.get(data_model_name='38/ClingenAlleleRegistry') + self.assertEqual(dv_38.version, '10') + diff --git a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py new file mode 100644 index 0000000000..4892a7221a --- /dev/null +++ b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py @@ -0,0 +1,424 @@ +import datetime +import gzip +import mock +import responses +from django.core.management import call_command +from django.core.management.base import CommandError +from django.forms.models import model_to_dict +from django.test import TestCase +from settings import SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL + +from clickhouse_search.management.commands.reload_clinvar_all_variants import BATCH_SIZE, WEEKLY_XML_RELEASE +from clickhouse_search.models.reference_data_models import ( + ClinvarSnvIndel, ClinvarGRCh37SnvIndel, ClinvarAllVariantsSnvIndel, + ClinvarAllVariantsGRCh37SnvIndel, ClinvarAllVariantsMito, +) +from reference_data.models import DataVersions +from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin + +WEEKLY_XML_RELEASE_HEADER = '''''' +WEEKLY_XML_RELEASE_DATA = WEEKLY_XML_RELEASE_HEADER + ''' + + + + + + + + + + + criteria provided, conflicting classifications + Conflicting classifications of pathogenicity; association; risk factor + Pathogenic(18); Likely pathogenic(9); Pathogenic, low penetrance(1); Established risk allele(1); Likely risk allele(1); Uncertain significance(1); VUS-high(1) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Pathogenic/Likely pathogenic/Pathogenic, low penetrance/Established risk allele + + + + + +''' + + +@mock.patch('clickhouse_search.management.commands.reload_clinvar_all_variants.safe_post_to_slack') +@mock.patch('clickhouse_search.management.commands.reload_clinvar_all_variants.logger.info') +class ReloadClinvarAllVariantsTest(DifferentDbTransactionSupportMixin, TestCase): + databases = '__all__' + fixtures = ['clinvar_all_variants'] + + @responses.activate + def test_update_with_no_previous_version(self, mock_logger, mock_safe_post_to_slack): + DataVersions.objects.all().delete() + ClinvarAllVariantsSnvIndel.objects.using('clickhouse_write').all().delete() + responses.add(responses.GET, WEEKLY_XML_RELEASE, status=200, body=gzip.compress(WEEKLY_XML_RELEASE_DATA.encode()), stream=True) + call_command('reload_clinvar_all_variants') + mock_logger.assert_called_with('Updating Clinvar ClickHouse tables to 2025-06-30 from None.') + self.assertEqual(ClinvarAllVariantsSnvIndel.objects.count(), 1) + + @responses.activate + def test_new_version_already_exists(self, mock_logger, mock_safe_post_to_slack): + version_obj = DataVersions.objects.filter(data_model_name='Clinvar').first() + version_obj.version = '2025-06-30' + version_obj.save() + responses.add(responses.GET, WEEKLY_XML_RELEASE, status=200, body=gzip.compress(WEEKLY_XML_RELEASE_DATA.encode()), stream=True) + call_command('reload_clinvar_all_variants') + mock_logger.assert_called_with('Clinvar ClickHouse tables already successfully updated to 2025-06-30, gracefully exiting.') + + @responses.activate + def test_parse_variants_all_types(self, mock_logger, mock_safe_post_to_slack): + responses.add(responses.GET, WEEKLY_XML_RELEASE, status=200, body=gzip.compress(WEEKLY_XML_RELEASE_DATA.encode()), stream=True) + call_command('reload_clinvar_all_variants') + mock_logger.assert_called_with('Updating Clinvar ClickHouse tables to 2025-06-30 from 2025-06-23.') + seqr_clinvar_snv_indel_models = ClinvarSnvIndel.objects.all() + self.assertEqual(seqr_clinvar_snv_indel_models.count(), 1) + self.assertDictEqual( + model_to_dict(seqr_clinvar_snv_indel_models.first()), + { + 'allele_id': 20642, + 'assertions': ['association', 'risk_factor'], + 'conditions': [ + 'Breast carcinoma', + 'CHEK2-related cancer predisposition', + 'CHEK2-related disorder', + 'Hereditary cancer-predisposing syndrome', + ], + 'conflicting_pathogenicities': [ + {'pathogenicity': 'Pathogenic', 'count': 19}, + {'pathogenicity': 'Likely_pathogenic', 'count': 9}, + {'pathogenicity': 'Uncertain_significance', 'count': 2}, + {'pathogenicity': 'Established_risk_allele', 'count': 1}, + {'pathogenicity': 'Likely_risk_allele', 'count': 1}, + ], + 'gold_stars': 1, + 'key': 12, + 'pathogenicity': 'Conflicting_classifications_of_pathogenicity', + 'submitters': [ + 'Fulgent Genetics, Fulgent Genetics', + 'Revvity Omics, Revvity', + 'University of Washington Department of Laboratory Medicine, University of Washington' + ] + }, + ) + + seqr_clinvar_grch37_snv_indel_models = ClinvarGRCh37SnvIndel.objects.all() + self.assertEqual(seqr_clinvar_grch37_snv_indel_models.count(), 0) + clinvar_all_variants_grch37_snv_indel_models = ClinvarAllVariantsGRCh37SnvIndel.objects.all() + self.assertEqual(clinvar_all_variants_grch37_snv_indel_models.count(), 1) + clinvar_all_variants_mito_models = ClinvarAllVariantsMito.objects.all() + self.assertDictEqual( + model_to_dict(clinvar_all_variants_mito_models.first()), + { + 'allele_id': 20642, + 'assertions': ['low_penetrance'], + 'conditions': [], + 'conflicting_pathogenicities': None, + 'gold_stars': None, + 'pathogenicity': 'Pathogenic/Likely_pathogenic/Established_risk_allele', + 'submitters': [], + 'variant_id': 'M-123-G-A', + 'version': datetime.date(2025, 6, 30) + } + ) + + # Version in Postgres. + dv = DataVersions.objects.get(data_model_name='Clinvar') + self.assertEqual(dv.version, '2025-06-30') + mock_safe_post_to_slack.assert_called_with( + SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, + 'Successfully updated Clinvar ClickHouse tables to 2025-06-30.', + ) + + + @responses.activate + def test_batching(self, mock_logger, mock_safe_post_to_slack): + # Dynamically build many variants + data = WEEKLY_XML_RELEASE_HEADER + for i in range(BATCH_SIZE * 2 + 10): + data += f''' + + + + + + + + + + + + + + + ''' + data += '' + responses.add(responses.GET, WEEKLY_XML_RELEASE, status=200, body=gzip.compress(data.encode()), stream=True) + call_command('reload_clinvar_all_variants') + mock_logger.assert_called_with('Updating Clinvar ClickHouse tables to 2025-06-30 from 2025-06-23.') + self.assertEqual(ClinvarAllVariantsSnvIndel.objects.all().count(), BATCH_SIZE * 2 + 10) + self.assertEqual(ClinvarAllVariantsSnvIndel.objects.first().pathogenicity, ClinvarAllVariantsSnvIndel.CLINVAR_DEFAULT_PATHOGENICITY) + self.assertIsNone(ClinvarAllVariantsSnvIndel.objects.first().gold_stars) + + @responses.activate + def test_malformed_variants(self, mock_logger, mock_safe_post_to_slack): + # Test errors that should still raise (not unenumerated values) + for description, review_status, conflicting_pathogenicities, error_message in [ + ("Pathogenic", "unhandled", None, 'Found unexpected review status unhandled'), + ("Conflicting classifications of pathogenicity", None, "Pathogenic;", 'Failed to correctly parse conflicting pathogenicity counts: Pathogenic;'), + ("Conflicting classifications of pathogenicity", None, None, 'Failed to find the conflicting pathogenicities node'), + ]: + data = f''' + + + + + + + + + + + {description} + {f"{review_status}" if review_status else ""} + {f"{conflicting_pathogenicities}" if conflicting_pathogenicities else ""} + + + + + ''' + responses.add( + responses.GET, + WEEKLY_XML_RELEASE, + status=200, + body=gzip.compress(data.encode()), + stream=True, + ) + with self.assertRaisesMessage(CommandError, error_message): + call_command('reload_clinvar_all_variants') + + # Test that unenumerated values are now collected instead of raising + unenumerated_test_cases = [ + { + 'allele_id': 1, + 'description': 'Pathogenic-ey', + 'explanation': None, + 'alert_type': 'Assertion', + 'alert_value': 'Pathogenic-ey', + }, + { + 'allele_id': 2, + 'description': 'Pathogenic; but unknown assertion', + 'explanation': None, + 'alert_type': 'Assertion', + 'alert_value': 'but unknown assertion', + }, + { + 'allele_id': 3, + 'description': 'Conflicting classifications of pathogenicity', + 'explanation': 'Pathogenic(18); unhandled(1)', + 'alert_type': 'Conflicting Pathogenicity', + 'alert_value': 'unhandled', + }, + ] + data = WEEKLY_XML_RELEASE_HEADER + for case in unenumerated_test_cases: + data += f''' + + + + + + + + + + {case['description']} + {f"{case['explanation']}" if case['explanation'] else ""} + + + + + ''' + data += '' + responses.add( + responses.GET, + WEEKLY_XML_RELEASE, + status=200, + body=gzip.compress(data.encode()), + stream=True, + ) + DataVersions.objects.all().delete() + ClinvarAllVariantsSnvIndel.objects.using('clickhouse_write').all().delete() + call_command('reload_clinvar_all_variants') + # Command should succeed and collect all unenumerated values in single slack message + slack_message = mock_safe_post_to_slack.call_args[0][1] + self.assertIn(f'Found {len(unenumerated_test_cases)} unenumerated value(s) during parsing:', slack_message) + for case in unenumerated_test_cases: + self.assertIn(f"- {case['alert_type']}: '{case['alert_value']}' (Allele ID: {case['allele_id']})", slack_message) + + # Variants with missing/equivalent alleles and positions are skipped + for simple_allele_attrs, sequence_location_attrs in [ + # Case 1: Missing AlleleId in + ("", 'Assembly="GRCh38" Chr="1" positionVCF="1" referenceAlleleVCF="G" alternateAlleleVCF="A"'), + + # Case 2: Missing alternateAlleleVCF in + ('AlleleID="5603"', 'Assembly="GRCh38" Chr="1" positionVCF="1" referenceAlleleVCF="G"'), + + # Case 3: ref/alt are equal + ('AlleleID="5603"', 'Assembly="GRCh38" Chr="1" positionVCF="1" referenceAlleleVCF="G" alternateAlleleVCF="G"'), + ]: + data = f''' + + + + + + + + + + + + + + + + + + Pathogenic + criteria provided, conflicting classifications + + + + + ''' + responses.add( + responses.GET, + WEEKLY_XML_RELEASE, + status=200, + body=gzip.compress(data.encode()), + stream=True, + ) + DataVersions.objects.all().delete() + ClinvarAllVariantsSnvIndel.objects.using('clickhouse_write').all().delete() + call_command('reload_clinvar_all_variants') + self.assertEqual(ClinvarAllVariantsSnvIndel.objects.count(), 0) + self.assertEqual(ClinvarAllVariantsGRCh37SnvIndel.objects.count(), 0) + mock_safe_post_to_slack.assert_called_with( + SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, + 'Successfully updated Clinvar ClickHouse tables to 2025-06-30.', + ) + + @responses.activate + def test_conflicting_data_from_submitters(self, mock_logger, mock_safe_post_to_slack): + missing_conflicting_pathogenicities = WEEKLY_XML_RELEASE_HEADER + ''' + + + + + + + + + + conflicting data from submitters + + + + + + ''' + responses.add( + responses.GET, + WEEKLY_XML_RELEASE, + status=200, + body=gzip.compress(missing_conflicting_pathogenicities.encode()), + stream=True, + ) + ClinvarAllVariantsSnvIndel.objects.using('clickhouse_write').all().delete() + with self.assertRaisesMessage(CommandError, 'Failed to find the conflicting pathogenicities node'): + call_command('reload_clinvar_all_variants') + + conflicting_pathogenicities = WEEKLY_XML_RELEASE_HEADER + ''' + + + + + + + + + + conflicting data from submitters + + + + + + Uncertain significance(1), Likely benign (1) + + + + + + + + ''' + responses.add( + responses.GET, + WEEKLY_XML_RELEASE, + status=200, + body=gzip.compress(conflicting_pathogenicities.encode()), + stream=True, + ) + call_command('reload_clinvar_all_variants') + self.assertEqual(ClinvarAllVariantsSnvIndel.objects.count(), 1) + self.assertEqual(ClinvarAllVariantsSnvIndel.objects.first().pathogenicity, ClinvarAllVariantsSnvIndel.CLINVAR_CONFLICTING_CLASSICATIONS_OF_PATHOGENICITY) + mock_safe_post_to_slack.assert_called_with( + SEQR_SLACK_DATA_ALERTS_NOTIFICATION_CHANNEL, + 'Successfully updated Clinvar ClickHouse tables to 2025-06-30.', + ) From ad7d406501646ea51d647c3ace4fd1fe130875f9 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 17:00:59 -0400 Subject: [PATCH 109/119] change test order --- .../{search_tests.py => all_search_tests.py} | 0 .../tests/set_saved_variant_key_tests.py | 133 ++++++++++++++++++ .../tag_seqr_prioritized_variants_tests.py | 2 +- 3 files changed, 134 insertions(+), 1 deletion(-) rename clickhouse_search/{search_tests.py => all_search_tests.py} (100%) create mode 100644 clickhouse_search/management/tests/set_saved_variant_key_tests.py diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/all_search_tests.py similarity index 100% rename from clickhouse_search/search_tests.py rename to clickhouse_search/all_search_tests.py diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py new file mode 100644 index 0000000000..9b3d07f479 --- /dev/null +++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py @@ -0,0 +1,133 @@ +from django.core.management import call_command +import mock + +from seqr.models import Project, Dataset, SavedVariant +from seqr.views.utils.test_utils import AnvilAuthenticationTestCase + +MOCK_GCNV_DATA = [ + b'chr\tstart\tend\tname\tsample\tsample_fix\tsvtype\tGT\tCN\tNP\tQA\tQS\tQSE\tQSS\tploidy\tstrand\tvariant_name\tID\trmsstd\tdefragmented\tvaf\tvac\tlt100_raw_calls\tlt10_highQS_rare_calls\tPASS_SAMPLE\tPASS_FREQ\tPASS_QS\tHIGH_QUALITY\tgenes_any_overlap\tgenes_any_overlap_exonsPerGene\tgenes_any_overlap_totalExons\tgenes_strict_overlap\tgenes_strict_overlap_exonsPerGene\tgenes_strict_overlap_totalExons\tgenes_CG\tgenes_LOF\tgenes_any_overlap_Ensemble_ID\tidentical\tpartial_ovl\tany_ovl\tno_ovl\tsource', + b'chr10\t122599967\t122600191\tCASE_7_Y_cnv_57308\tC1933_010-001_v1_Exome_GCP\t010-001_v1_Exome_C1933\tDEL\t1\t1\t1\t7\t7\t7\t72\t\t-\tprefix_70191\t326536\t0\tFALSE\t0.05066805\t1077\tTRUE\tTRUE\tTRUE\tFALSE\tFALSE\tFALSE\tDMBT1\t1\t1\tNA\t0\t0\tNA\tDMBT1\tENSG00000187908.19\tcluster_10_CASE_cnv_46117\tcluster_10_CASE_cnv_46117\tcluster_10_CASE_cnv_46117;prefix_19107\tFALSE\tround2', +] + + +class SetSavedVariantKeyTest(AnvilAuthenticationTestCase): + fixtures = ['users', '1kg_project', 'report_variants', 'clickhouse_saved_variants'] + + MOCK_GCNV_DATA = MOCK_GCNV_DATA + + @classmethod + def setUpTestData(cls): + Project.objects.filter(id=3).update(genome_version='38') + dataset = Dataset.objects.get(guid='S000154_na20889') + dataset.dataset_type = 'SV' + dataset.save() + dataset.active_individuals.set({17}) + for sv in SavedVariant.objects.filter(key__isnull=False): + sv.saved_variant_json = { + 'genotypes': sv.genotypes, 'populations': {'gnomad': {'af': 0.01}}, + } + if sv.guid == 'SV0000009_25000014783_r0004_no': + sv.saved_variant_json['populations']['seqr'] = {'af': 0.019480518996715546, 'ac': 3, 'an': 154} + sv.save() + SavedVariant.objects.update(key=None) + SavedVariant.objects.filter(guid='SV0000007_prefix_19107_DEL_r00').update(variant_id='prefix_19107_DEL') + + @mock.patch('clickhouse_search.management.commands.set_saved_variant_key.BATCH_SIZE', 2) + @mock.patch('seqr.utils.file_utils.subprocess.Popen') + def test_command(self, mock_subprocess): + mock_subprocess.return_value.stdout = self.MOCK_GCNV_DATA + mock_subprocess.return_value.wait.return_value = 0 + + call_command('set_saved_variant_key') + self.assert_json_logs(user=None, expected=[ + ('Finding keys for 1 MITO (GRCh38) variant ids', None), + ('Found 1 keys', None), + ('Updated batch of 1', None), + ('Updated keys for 1 MITO (GRCh38) variants', None), + ('Finding keys for 1 SNV_INDEL (GRCh38) variant ids', None), + ('Found 1 keys', None), + ('Updated batch of 2', None), + ('Updated keys for 2 SNV_INDEL (GRCh38) variants', None), + ('Finding keys for 2 SV_WGS (GRCh38) variant ids', None), + ('Found 1 keys', None), + ('Updated batch of 1', None), + ('Updated keys for 1 SV_WGS (GRCh38) variants', None), + ('No key found for 1 variants', None), + ('Finding keys for 1 SV_WES (GRCh38) variant ids', None), + ('Found 1 keys', None), + ('Updated batch of 1', None), + ('Updated keys for 1 SV_WES (GRCh38) variants', None), + ('Finding keys for 7 SNV_INDEL (GRCh37) variant ids', None), + ('Found 1 keys', None), + ('Updated batch of 1', None), + ('Updated keys for 1 SNV_INDEL (GRCh37) variants', None), + ('No key found for 6 variants', None), + ('6 variants have no key, 0 of which have no search data, 6 of which are absent from the hail backend.', None), + ('Cleared saved json for 6 variants with keys', None), + ('Done', None), + ]) + + saved_variants = list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id', 'saved_variant_json')) + expected_saved_variants = [ + {'guid': 'SV0000001_2103343353_r0390_100', 'key': None, 'variant_id': '21-3343353-GAGA-G', 'saved_variant_json': mock.ANY}, + {'guid': 'SV0000002_1248367227_r0390_100', 'key': 100, 'variant_id': '1-248367227-TC-T', 'saved_variant_json': {}}, + {'guid': 'SV0000006_1248367227_r0003_tes', 'key': 100, 'variant_id': '1-248367227-TC-T', 'saved_variant_json': {}}, + {'guid': 'SV0000006_1248367227_r0004_non', 'key': 100, 'variant_id': '1-248367227-TC-T', 'saved_variant_json': {}}, + {'guid': 'SV0000007_prefix_19107_DEL_r00', 'key': 111, 'variant_id': 'prefix_19107_DEL', 'saved_variant_json': {}}, + {'guid': 'SV0000009_25000014783_r0004_no', 'key': 100, 'variant_id': 'M-14783-T-C', 'saved_variant_json': {}}, + {'guid': 'SV0000013_prefix_19107_DEL_r00', 'key': 101, 'variant_id': 'suffix_19107_DEL_013746', 'saved_variant_json': {}}, + {'guid': 'SV0027166_191912634_r0384_rare', 'key': None, 'variant_id': '19-1912634-C-T', 'saved_variant_json': mock.ANY}, + {'guid': 'SV0027167_191912633_r0384_rare', 'key': None, 'variant_id': '19-1912633-G-T', 'saved_variant_json': mock.ANY}, + {'guid': 'SV0027168_191912632_r0384_rare', 'key': None, 'variant_id': '19-1912632-G-C', 'saved_variant_json': mock.ANY}, + {'guid': 'SV0059956_11560662_f019313_1', 'key': None, 'variant_id': '1-46859832-G-A', 'saved_variant_json': mock.ANY}, + {'guid': 'SV0059957_11562437_f019313_1', 'key': None, 'variant_id': '1-1562437-G-CA', 'saved_variant_json': mock.ANY}, + ] + self.assertListEqual(saved_variants, expected_saved_variants) + + # Reloading is a no-op + self.reset_logs() + call_command('set_saved_variant_key') + self.assert_json_logs(user=None, expected=[ + ('Finding keys for 6 SNV_INDEL (GRCh37) variant ids', None), + ('Found 0 keys', None), + ('6 variants have no key, 0 of which have no search data, 6 of which are absent from the hail backend.', None), + ('Cleared saved json for 0 variants with keys', None), + ('Done', None), + ]) + self.assertListEqual( + list(SavedVariant.objects.order_by('guid').values('guid', 'key', 'variant_id', 'saved_variant_json')), + expected_saved_variants, + ) + +class SetSavedVariantKeyFailedMappingTest(SetSavedVariantKeyTest): + fixtures = ['users', '1kg_project', 'report_variants'] + + MOCK_GCNV_DATA = MOCK_GCNV_DATA[:1] + + @mock.patch('seqr.utils.file_utils.subprocess.Popen') + def test_command(self, mock_subprocess): + mock_subprocess.return_value.stdout = self.MOCK_GCNV_DATA + mock_subprocess.return_value.wait.return_value = 0 + + call_command('set_saved_variant_key') + self.assert_json_logs(user=None, expected=[ + ('Finding keys for 1 MITO (GRCh38) variant ids', None), + ('Found 0 keys', None), + ('Finding keys for 2 SNV_INDEL (GRCh38) variant ids', None), + ('Found 0 keys', None), + ('3 variants have no key, 1 of which have no search data, 1 of which are absent from the hail backend.', None), + ('1 remaining variants: M-14783-T-C - fam14', None), + ('Finding keys for 2 SV_WGS (GRCh38) variant ids', None), + ('Found 0 keys', None), + ('Finding keys for 2 SV_WES (GRCh38) variant ids', None), + ('Found 0 keys', None), + ('2 SV variants have no key, 0 of which have no search data', None), + ('1 remaining SV WES variants prefix_19107_DEL - 12', None), + ('1 remaining SV WGS variants suffix_19107_DEL_013746 - fam14', None), + ('Finding keys for 7 SNV_INDEL (GRCh37) variant ids', None), + ('Found 0 keys', None), + ('7 variants have no key, 0 of which have no search data, 7 of which are absent from the hail backend.', None), + ('Cleared saved json for 0 variants with keys', None), + ('Done', None), + ]) + diff --git a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py index f03bd95a59..34c77cf5e1 100644 --- a/seqr/management/tests/tag_seqr_prioritized_variants_tests.py +++ b/seqr/management/tests/tag_seqr_prioritized_variants_tests.py @@ -3,7 +3,7 @@ import json import mock -from clickhouse_search.search_tests import ClickhouseSearchTestCase +from clickhouse_search.all_search_tests import ClickhouseSearchTestCase from clickhouse_search.test_utils import VARIANT2, VARIANT3, VARIANT4, GCNV_VARIANT3, GCNV_VARIANT4 from seqr.models import SavedVariant, VariantTag From a3d04c0ab04d09e3aef1f7f385571142678f038d Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 17:06:10 -0400 Subject: [PATCH 110/119] revert unrelated test changes --- .../management/tests/register_caids_tests.py | 42 +++++++++++++++++-- .../reload_clinvar_all_variants_tests.py | 4 +- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/clickhouse_search/management/tests/register_caids_tests.py b/clickhouse_search/management/tests/register_caids_tests.py index 963633e431..d0c27f257f 100644 --- a/clickhouse_search/management/tests/register_caids_tests.py +++ b/clickhouse_search/management/tests/register_caids_tests.py @@ -11,7 +11,6 @@ VariantDetailsSnvIndel, ) from reference_data.models import DataVersions -from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin MOCK_RESPONSE = [ { @@ -51,6 +50,26 @@ }, ], }, + { + '@id': 'http://reg.genome.network/allele/CA16716504', + 'genomicAlleles': [ + { + 'chromosome': '1', + 'coordinates': [ + { + 'allele': 'C', + 'end': 10131, + 'referenceAllele': '', # ref allele is '' + 'start': 10131, + }, + ], + 'referenceGenome': 'GRCh38', + }, + ], + 'externalRecords': { + 'gnomAD_4': [{'id': '1-91511686-91511734'}], # has invalid gnomad ID + }, + }, { '@id': 'http://reg.genome.network/allele/CA997563845', 'genomicAlleles': [ @@ -95,7 +114,7 @@ @mock.patch('clickhouse_search.management.commands.register_caids.logger') @mock.patch("clickhouse_search.management.commands.register_caids.safe_post_to_slack") -class RegisterCaidsEmptyDatabaseTest(DifferentDbTransactionSupportMixin, TestCase): +class RegisterCaidsEmptyDatabaseTest(TestCase): databases = "__all__" fixtures = [] @@ -114,7 +133,7 @@ def test_register_caids(self, mock_safe_post_to_slack, mock_logger): @mock.patch('clickhouse_search.management.commands.register_caids.logger') @mock.patch("clickhouse_search.management.commands.register_caids.safe_post_to_slack") -class RegisterCaidsTest(DifferentDbTransactionSupportMixin, TestCase): +class RegisterCaidsTest(TestCase): databases = "__all__" fixtures = ["variant_details_for_update"] @@ -136,6 +155,21 @@ def test_bad_responses(self, mock_safe_post_to_slack, mock_logger): call_command("register_caids", batch_size=5) mock_safe_post_to_slack.assert_not_called() + responses.add( + responses.PUT, + "https://reg.genome.network/alleles", + match=[ + responses.matchers.query_param_matcher({ + "file": "vcf", + "fields": "none @id genomicAlleles externalRecords.gnomAD_4.id", + }, strict_match=False), + ], + status=400, + ) + with self.assertRaisesMessage(CommandError, 'Failed in 38/ClingenAlleleRegistry curr_key: 3'): + call_command("register_caids", batch_size=5) + mock_safe_post_to_slack.assert_not_called() + responses.reset() mock_safe_post_to_slack.reset_mock() mock_logger.reset_mock() @@ -225,7 +259,7 @@ def test_register_caids(self, mock_safe_post_to_slack, mock_logger): dv_38 = DataVersions.objects.get(data_model_name='38/ClingenAlleleRegistry') self.assertEqual(dv_38.version, '10') mock_logger.info.assert_called_with( - "2 registered variant(s) cannot be mapped back to ours. \n" + "3 registered variant(s) cannot be mapped back to ours. \n" "First unmappable variant:\n{'@id': 'http://reg.genome.network/allele/CA16716503', 'genomicAlleles': [{'chromosome': '1', 'coordinates': [{'allele': 'C', 'end': 10131, 'referenceAllele': '', 'start': 10131}], 'referenceGenome': 'GRCh38'}]}" ) mock_logger.warning.assert_called_with( diff --git a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py index 4892a7221a..c22bff5763 100644 --- a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py +++ b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py @@ -14,7 +14,6 @@ ClinvarAllVariantsGRCh37SnvIndel, ClinvarAllVariantsMito, ) from reference_data.models import DataVersions -from seqr.views.utils.test_utils import DifferentDbTransactionSupportMixin WEEKLY_XML_RELEASE_HEADER = '''''' WEEKLY_XML_RELEASE_DATA = WEEKLY_XML_RELEASE_HEADER + ''' @@ -83,7 +82,7 @@ @mock.patch('clickhouse_search.management.commands.reload_clinvar_all_variants.safe_post_to_slack') @mock.patch('clickhouse_search.management.commands.reload_clinvar_all_variants.logger.info') -class ReloadClinvarAllVariantsTest(DifferentDbTransactionSupportMixin, TestCase): +class ReloadClinvarAllVariantsTest(TestCase): databases = '__all__' fixtures = ['clinvar_all_variants'] @@ -379,7 +378,6 @@ def test_conflicting_data_from_submitters(self, mock_logger, mock_safe_post_to_s body=gzip.compress(missing_conflicting_pathogenicities.encode()), stream=True, ) - ClinvarAllVariantsSnvIndel.objects.using('clickhouse_write').all().delete() with self.assertRaisesMessage(CommandError, 'Failed to find the conflicting pathogenicities node'): call_command('reload_clinvar_all_variants') From 3d98faec31a2f68362ba960109ec68fad406db22 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 17:08:32 -0400 Subject: [PATCH 111/119] revert unrelated test changes --- .../management/tests/register_caids_tests.py | 37 +------------------ .../reload_clinvar_all_variants_tests.py | 2 +- 2 files changed, 2 insertions(+), 37 deletions(-) diff --git a/clickhouse_search/management/tests/register_caids_tests.py b/clickhouse_search/management/tests/register_caids_tests.py index d0c27f257f..9985799c50 100644 --- a/clickhouse_search/management/tests/register_caids_tests.py +++ b/clickhouse_search/management/tests/register_caids_tests.py @@ -50,26 +50,6 @@ }, ], }, - { - '@id': 'http://reg.genome.network/allele/CA16716504', - 'genomicAlleles': [ - { - 'chromosome': '1', - 'coordinates': [ - { - 'allele': 'C', - 'end': 10131, - 'referenceAllele': '', # ref allele is '' - 'start': 10131, - }, - ], - 'referenceGenome': 'GRCh38', - }, - ], - 'externalRecords': { - 'gnomAD_4': [{'id': '1-91511686-91511734'}], # has invalid gnomad ID - }, - }, { '@id': 'http://reg.genome.network/allele/CA997563845', 'genomicAlleles': [ @@ -155,21 +135,6 @@ def test_bad_responses(self, mock_safe_post_to_slack, mock_logger): call_command("register_caids", batch_size=5) mock_safe_post_to_slack.assert_not_called() - responses.add( - responses.PUT, - "https://reg.genome.network/alleles", - match=[ - responses.matchers.query_param_matcher({ - "file": "vcf", - "fields": "none @id genomicAlleles externalRecords.gnomAD_4.id", - }, strict_match=False), - ], - status=400, - ) - with self.assertRaisesMessage(CommandError, 'Failed in 38/ClingenAlleleRegistry curr_key: 3'): - call_command("register_caids", batch_size=5) - mock_safe_post_to_slack.assert_not_called() - responses.reset() mock_safe_post_to_slack.reset_mock() mock_logger.reset_mock() @@ -259,7 +224,7 @@ def test_register_caids(self, mock_safe_post_to_slack, mock_logger): dv_38 = DataVersions.objects.get(data_model_name='38/ClingenAlleleRegistry') self.assertEqual(dv_38.version, '10') mock_logger.info.assert_called_with( - "3 registered variant(s) cannot be mapped back to ours. \n" + "2 registered variant(s) cannot be mapped back to ours. \n" "First unmappable variant:\n{'@id': 'http://reg.genome.network/allele/CA16716503', 'genomicAlleles': [{'chromosome': '1', 'coordinates': [{'allele': 'C', 'end': 10131, 'referenceAllele': '', 'start': 10131}], 'referenceGenome': 'GRCh38'}]}" ) mock_logger.warning.assert_called_with( diff --git a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py index c22bff5763..3815f6ef4a 100644 --- a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py +++ b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py @@ -84,7 +84,7 @@ @mock.patch('clickhouse_search.management.commands.reload_clinvar_all_variants.logger.info') class ReloadClinvarAllVariantsTest(TestCase): databases = '__all__' - fixtures = ['clinvar_all_variants'] + fixtures = ['test_conflicting_data_from_submitters'] @responses.activate def test_update_with_no_previous_version(self, mock_logger, mock_safe_post_to_slack): From 11c1dd777cccd90e993131e80d98d998d1c04f9a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 17:08:54 -0400 Subject: [PATCH 112/119] revert unrelated test changes --- .../management/tests/reload_clinvar_all_variants_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py index 3815f6ef4a..c22bff5763 100644 --- a/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py +++ b/clickhouse_search/management/tests/reload_clinvar_all_variants_tests.py @@ -84,7 +84,7 @@ @mock.patch('clickhouse_search.management.commands.reload_clinvar_all_variants.logger.info') class ReloadClinvarAllVariantsTest(TestCase): databases = '__all__' - fixtures = ['test_conflicting_data_from_submitters'] + fixtures = ['clinvar_all_variants'] @responses.activate def test_update_with_no_previous_version(self, mock_logger, mock_safe_post_to_slack): From 2dcec5819d7597036b016b8cfdb058b716af0101 Mon Sep 17 00:00:00 2001 From: hanars Date: Mon, 11 May 2026 17:11:21 -0400 Subject: [PATCH 113/119] run all tests --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 825fff6cdc..a032b6daf0 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -99,7 +99,7 @@ jobs: export CLICKHOUSE_READER_USER=clickhouse_read_only export CLICKHOUSE_WRITER_USER=clickhouse_test export CLICKHOUSE_SERVICE_HOSTNAME=localhost - coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 clickhouse_search + coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data seqr matchmaker panelapp clickhouse_search coverage report | (! grep [1-8][0-9]%) coverage report | (! grep 9[0-5]%) From 4784e359954435fc12b844e92ec1fb60050c506f Mon Sep 17 00:00:00 2001 From: hanars Date: Mon, 11 May 2026 17:12:13 -0400 Subject: [PATCH 114/119] fix accidental delete --- .github/workflows/unit-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index a032b6daf0..6e138c1ca1 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -100,6 +100,7 @@ jobs: export CLICKHOUSE_WRITER_USER=clickhouse_test export CLICKHOUSE_SERVICE_HOSTNAME=localhost coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data seqr matchmaker panelapp clickhouse_search + coverage report -m --fail-under=99 coverage report | (! grep [1-8][0-9]%) coverage report | (! grep 9[0-5]%) From 2a5e8975c1e2f024eac301149d06efc41bf05829 Mon Sep 17 00:00:00 2001 From: hanars Date: Mon, 11 May 2026 17:23:56 -0400 Subject: [PATCH 115/119] try skipping tests --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6e138c1ca1..6c225944d4 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -99,7 +99,7 @@ jobs: export CLICKHOUSE_READER_USER=clickhouse_read_only export CLICKHOUSE_WRITER_USER=clickhouse_test export CLICKHOUSE_SERVICE_HOSTNAME=localhost - coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data seqr matchmaker panelapp clickhouse_search + coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data matchmaker panelapp clickhouse_search coverage report -m --fail-under=99 coverage report | (! grep [1-8][0-9]%) coverage report | (! grep 9[0-5]%) From a7d3ce946b26bb4b7838ff7f2f403c939653960f Mon Sep 17 00:00:00 2001 From: hanars Date: Mon, 11 May 2026 17:50:05 -0400 Subject: [PATCH 116/119] run all tests --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6c225944d4..6e138c1ca1 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -99,7 +99,7 @@ jobs: export CLICKHOUSE_READER_USER=clickhouse_read_only export CLICKHOUSE_WRITER_USER=clickhouse_test export CLICKHOUSE_SERVICE_HOSTNAME=localhost - coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data matchmaker panelapp clickhouse_search + coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data seqr matchmaker panelapp clickhouse_search coverage report -m --fail-under=99 coverage report | (! grep [1-8][0-9]%) coverage report | (! grep 9[0-5]%) From 7e3c86100ff82b4dfeef6ec086d561b52e40a8f6 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 17:51:09 -0400 Subject: [PATCH 117/119] debug no dict refresh --- seqr/views/utils/individual_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index 6007ff8d6d..16aa680974 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -71,6 +71,7 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge updated_family_models = Family.objects.filter(id__in=updated_family_ids) _remove_pedigree_images(updated_family_models, user) if updated_affected: + raise ValuError('Nope!') AffectedDict.reload(user) if not skip_gt_stats_rebuild: trigger_rebuild_gt_stats(project, user) From 7dde22ccd02bccf2263dadfb8a4e6a265b58e243 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 11 May 2026 18:03:27 -0400 Subject: [PATCH 118/119] undo debug --- seqr/views/utils/individual_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index 16aa680974..6007ff8d6d 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -71,7 +71,6 @@ def add_or_update_individuals_and_families(project, individual_records, user, ge updated_family_models = Family.objects.filter(id__in=updated_family_ids) _remove_pedigree_images(updated_family_models, user) if updated_affected: - raise ValuError('Nope!') AffectedDict.reload(user) if not skip_gt_stats_rebuild: trigger_rebuild_gt_stats(project, user) From 6993e92ef8e7c7791035ab6856a0dde9ce9f9809 Mon Sep 17 00:00:00 2001 From: hanars Date: Mon, 11 May 2026 18:04:21 -0400 Subject: [PATCH 119/119] test order --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6e138c1ca1..daf1dfe876 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -99,7 +99,7 @@ jobs: export CLICKHOUSE_READER_USER=clickhouse_read_only export CLICKHOUSE_WRITER_USER=clickhouse_test export CLICKHOUSE_SERVICE_HOSTNAME=localhost - coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data seqr matchmaker panelapp clickhouse_search + coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp","./clickhouse_search" --omit="*/migrations/*","*/apps.py","*/admin.py" manage.py test -p '*_tests.py' -v 2 reference_data clickhouse_search seqr matchmaker panelapp coverage report -m --fail-under=99 coverage report | (! grep [1-8][0-9]%) coverage report | (! grep 9[0-5]%)