diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py index 2569dfc3a8..c76769effd 100644 --- a/clickhouse_search/management/commands/set_saved_variant_key.py +++ b/clickhouse_search/management/commands/set_saved_variant_key.py @@ -2,13 +2,12 @@ from django.contrib.postgres.aggregates import ArrayAgg from django.core.management.base import BaseCommand -from django.db.models import F import logging from clickhouse_search.search import get_clickhouse_key_lookup from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 from seqr.models import SavedVariant, Sample -from seqr.utils.search.utils import parse_variant_id +from seqr.utils.xpos_utils import parse_variant_id logger = logging.getLogger(__name__) diff --git a/clickhouse_search/managers.py b/clickhouse_search/managers.py index b514d2620d..c9293b7924 100644 --- a/clickhouse_search/managers.py +++ b/clickhouse_search/managers.py @@ -19,9 +19,12 @@ CLINVAR_CONFLICTING_P_LP, CLINVAR_CONFLICTING_NO_P, CLINVAR_CONFLICTING, PATH_FREQ_OVERRIDE_CUTOFF, \ HGMD_CLASS_FILTERS, SV_TYPE_FILTER_FIELD, SV_CONSEQUENCES_FIELD, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS, \ X_LINKED_RECESSIVE_MALE_AFFECTED, FEMALE_SEXES, SV_ANNOTATION_TYPES -from seqr.utils.xpos_utils import get_xpos, MIN_POS, MAX_POS, CHROMOSOME_CHOICES +from seqr.utils.xpos_utils import get_xpos, parse_variant_id, MIN_POS, MAX_POS, CHROMOSOME_CHOICES +class InvalidSearchException(Exception): + pass + class InvalidDatasetTypeException(Exception): pass @@ -96,11 +99,6 @@ def _annotate_seqr_pop_expression(self, results): return results return results.annotate(seqrPop=self.gt_stats_dict.dict_get_expression('key')) - def _format_gene_intervals(self, genes): - return [ - {field: gene[f'{field}Grch{self.genome_version}'] for field in ['chrom', 'start', 'end']} for gene in genes.values() - ] - @classmethod def _prediction_expression(cls, model): pred_expressions = [] @@ -418,12 +416,8 @@ def populations(self): def _filter_frequency(self, results, **kwargs): return results - def _filter_annotations(self, results, annotations_filter_q=None, transcript_filters=None, genes=None, intervals=None, exclude_locations=False, **kwargs): - if exclude_locations and genes: - intervals = self._format_gene_intervals(genes) - genes = None - - results = self._filter_locations(results, genes, intervals, exclude_locations=exclude_locations, **kwargs) + def _filter_annotations(self, results, annotations_filter_q=None, transcript_filters=None, genes=None, **kwargs): + results = self._filter_locations(results, genes, **kwargs) if not (annotations_filter_q or transcript_filters): return results @@ -448,7 +442,7 @@ def _annotate_filtered_transcripts(self, results, consequence_field, transcript_ self.FILTERED_CONSEQUENCE_FIELD: ArrayFilter(consequence_field, conditions=transcript_filters), }) - def _filter_locations(self, results, genes, intervals, **kwargs): + def _filter_locations(self, results, genes, intervals=None, **kwargs): if genes: results = results.annotate(**{ self.GENE_CONSEQUENCE_FIELD: ArrayFilter(self.TRANSCRIPT_FIELD, conditions=[{ @@ -608,13 +602,18 @@ def _parse_in_silico_qs(self, results, in_silico, require_score, in_silico_qs, i return results, False, in_silico_q, missing_q - def _filter_annotations(self, results, *args, exclude=None, **kwargs): + def _filter_annotations(self, results, *args, exclude=None, pathogenicity=None, **kwargs): screen_expression = self._screen_expression() if screen_expression: results = results.annotate(screen=self._screen_expression()) results = super()._filter_annotations(results, *args, **kwargs) + if (exclude or {}).get(CLINVAR_KEY) and (pathogenicity or {}).get(CLINVAR_KEY): + duplicates = set(pathogenicity[CLINVAR_KEY]).intersection(exclude[CLINVAR_KEY]) + if duplicates: + raise InvalidSearchException(f'ClinVar pathogenicity {", ".join(sorted(duplicates))} is both included and excluded') + exclude_clinvar_q = self._clinvar_filter_q(exclude) if exclude_clinvar_q is not None: results = results.exclude(exclude_clinvar_q) @@ -835,7 +834,7 @@ def _parse_annotation_filters(self, annotations, pathogenicity): return filter_qs, transcript_filters - def _filter_locations(self, results, genes, intervals, exclude_locations=False, padded_interval_end=None, **kwargs): + def _filter_locations(self, results, genes, intervals=None, exclude_intervals=None, padded_interval_end=None, **kwargs): results = super()._filter_locations(results, genes, intervals, **kwargs) if genes: @@ -844,7 +843,7 @@ def _filter_locations(self, results, genes, intervals, exclude_locations=False, end, padding = padded_interval_end interval_qs = [Q(end__range=(end - padding, end + padding))] else: - interval_qs = [self._interval_query(**interval) for interval in (intervals or [])] + interval_qs = [self._interval_query(**interval) for interval in (intervals or exclude_intervals or [])] if not interval_qs: return results @@ -852,7 +851,7 @@ def _filter_locations(self, results, genes, intervals, exclude_locations=False, interval_q = interval_qs[0] for q in interval_qs[1:]: interval_q |= q - filter_func = results.exclude if exclude_locations else results.filter + filter_func = results.exclude if exclude_intervals else results.filter return filter_func(interval_q) def _interval_query(self, chrom, start, end, *args, **kwargs): @@ -1036,7 +1035,7 @@ def search(self, sample_data, exclude_keys=None, **kwargs): entries = entries.exclude(key__in=exclude_keys) entries = self._join_annotations(entries) - entries = self._prefilter_entries(entries, **kwargs) + entries = self._prefilter_entries(entries, sample_data=sample_data, **kwargs) return self._search_call_data(entries, sample_data, **kwargs) @@ -1090,6 +1089,8 @@ def _search_call_data(self, entries, sample_data, inheritance_mode=None, inherit quality_q = None gt_filter = None quality_filter = qualityFilter or {} + if not inheritance_mode and list((inheritance_filter or {}).keys()) == ['affected']: + raise InvalidSearchException('Inheritance must be specified if custom affected status is set') if inheritance_mode or (inheritance_filter or {}).get('genotype') or quality_filter: clinvar_override_q = self._clinvar_path_q(pathogenicity) inheritance_q, quality_q, gt_filter, carriers_expression = self._get_inheritance_quality_qs( @@ -1431,7 +1432,7 @@ def _family_passes_expression(pass_field): mapped_expression='x.1', output_field=models.ArrayField(models.StringField()), ) - def filter_locus(self, exclude_locations=False, require_gene_filter=False, intervals=None, genes=None, variant_ids=None, inheritance_mode=None, **kwargs): + def filter_locus(self, exclude_intervals=None, require_gene_filter=False, intervals=None, genes=None, variant_ids=None, inheritance_mode=None, **kwargs): entries = self if variant_ids: @@ -1444,20 +1445,30 @@ def filter_locus(self, exclude_locations=False, require_gene_filter=False, inter if intervals and self.filtered_chrom: intervals = [interval for interval in intervals if interval['chrom'] == self.filtered_chrom] - if not intervals and not exclude_locations: + if not intervals: raise InvalidDatasetTypeException if genes or intervals: - entries = self._filter_locations(entries, genes, intervals, exclude_locations=exclude_locations, require_gene_filter=require_gene_filter) + entries = entries.filter(self._filter_locations_q(intervals, genes, require_gene_filter)) + elif exclude_intervals: + entries = entries.exclude(self._filter_locations_q(exclude_intervals)) return entries - def _filter_locations(self, entries, genes, intervals, exclude_locations=False, require_gene_filter=False): + @staticmethod + def _parse_variant_ids(raw_variant_items): + parsed_variant_ids = {} + for item in (raw_variant_items or '').replace(',', ' ').split(): + variant_id = item.replace('chr', '') + parsed_variant_ids[variant_id] = parse_variant_id(variant_id) + return parsed_variant_ids + + def _filter_locations_q(self, intervals, genes=None, require_gene_filter=False): locus_q = None if genes: - should_filter_interval = self._can_filter_gene_interval(genes) or exclude_locations + should_filter_interval = self._can_filter_gene_interval(genes) if should_filter_interval: - intervals = self._format_gene_intervals(genes) + (intervals or []) + intervals = list((genes or {}).values()) + (intervals or []) if require_gene_filter or (not should_filter_interval): locus_q = Q(geneId_ids__bitmap_has_any=[gene['id'] for gene in genes.values()]) @@ -1472,8 +1483,7 @@ def _filter_locations(self, entries, genes, intervals, exclude_locations=False, else: locus_q |= interval_q - filter_func = entries.exclude if exclude_locations else entries.filter - return filter_func(locus_q) + return locus_q def _can_filter_gene_interval(self, genes): return (not hasattr(self.model, 'geneId_ids')) or len(genes) < self.MAX_XPOS_FILTER_INTERVALS or self.filtered_chrom @@ -1507,6 +1517,7 @@ def _filter_seqr_freq_field(self, entries, freq_field, value): class EntriesManager(BaseEntriesManager): + MIN_MULTI_FAMILY_SEQR_AC = 5000 @staticmethod def _clinvar_range_q(path_range): @@ -1533,7 +1544,15 @@ def _prefilter_entries(self, entries, freqs=None, **kwargs): entries = self._filter_in_silico(entries, **kwargs) return entries - def _filter_frequency(self, entries, frequencies, pathogenicity=None, **kwargs): + def _filter_frequency(self, entries, frequencies, pathogenicity=None, sample_data=None, **kwargs): + callset_filter = frequencies.get(self.callset_filter_field) or {} + if (not callset_filter.get('ac') or callset_filter['ac'] >= self.MIN_MULTI_FAMILY_SEQR_AC) and ( + len(sample_data['sample_type_families']) > 1 or len(next(iter(sample_data['sample_type_families'].values()))) > 1 + ): + raise InvalidSearchException( + f'seqr AC frequency of at least {self.MIN_MULTI_FAMILY_SEQR_AC} must be specified to search across multiple families' + ) + gnomad_filter = frequencies.get('gnomad_genomes') or {} if hasattr(self.model, 'is_gnomad_gt_5_percent') and ( (gnomad_filter.get('af') or 1) <= 0.05 or any(gnomad_filter.get(field) is not None for field in ['ac', 'hh']) @@ -1636,14 +1655,18 @@ def _join_annotations(self, entries): ) return super()._join_annotations(entries) - def filter_locus(self, *args, require_any_gene=False, parsed_variant_ids=None, intervals=None, genes=None, variant_ids=None, **kwargs): - if parsed_variant_ids: + def filter_locus(self, *args, require_any_gene=False, raw_variant_items=None, intervals=None, genes=None, variant_ids=None, **kwargs): + if raw_variant_items: + parsed_variant_ids = self._parse_variant_ids(raw_variant_items) + invalid_items = [variant_id for variant_id, parsed_id in parsed_variant_ids.items() if not parsed_id] + if invalid_items: + if variant_ids: + raise InvalidDatasetTypeException + raise InvalidSearchException('Invalid variants: {}'.format(', '.join(invalid_items))) # although technically redundant, the interval query is applied to the entries table to # improve performance by using the xpos projection - intervals = [{'chrom': chrom, 'start': pos, 'end': pos} for chrom, pos, _, _ in parsed_variant_ids] - variant_ids = ['-'.join([str(o) for o in variant_id]) for variant_id in parsed_variant_ids] - elif variant_ids: - raise InvalidDatasetTypeException + intervals = [{'chrom': chrom, 'start': pos, 'end': pos} for chrom, pos, _, _ in parsed_variant_ids.values()] + variant_ids = parsed_variant_ids.keys() entries = super().filter_locus(*args, intervals=intervals, genes=genes, variant_ids=variant_ids, **kwargs) @@ -1715,24 +1738,19 @@ def _annotate_calls(self, entries, *args, **kwargs): return entries - def filter_locus(self, *args, exclude_locations=False, intervals=None, genes=None, exclude_svs=False, parsed_variant_ids=None, **kwargs): - if exclude_svs or parsed_variant_ids: + def filter_locus(self, *args, exclude_intervals=None, intervals=None, genes=None, exclude_svs=False, raw_variant_items=None, **kwargs): + if exclude_svs or any(self._parse_variant_ids(raw_variant_items).values()): raise InvalidDatasetTypeException # SV interval filtering occurs after joining on annotations to correctly incorporate end position can_filter_gene_interval = self._can_filter_gene_interval(genes) - if exclude_locations: - intervals = None - else: - chromosomes = {i['chrom'] for i in (intervals or [])} - if can_filter_gene_interval: - chromosomes.update({gene[f'chromGrch{self.genome_version}'] for gene in genes.values()}) - intervals = [{'chrom': chrom, 'start': MIN_POS, 'end': MAX_POS} for chrom in chromosomes] + chromosomes = {i['chrom'] for i in (intervals or [])} + if can_filter_gene_interval: + chromosomes.update({gene['chrom'] for gene in genes.values()}) + intervals = [{'chrom': chrom, 'start': MIN_POS, 'end': MAX_POS} for chrom in chromosomes] - entries = super().filter_locus( - *args, exclude_locations=exclude_locations, intervals=intervals, genes=genes, **kwargs, - ) + entries = super().filter_locus(*args, intervals=intervals, genes=genes, **kwargs) - if can_filter_gene_interval and not exclude_locations: + if can_filter_gene_interval: entries = entries.filter(calls__array_all={'OR': [ {'geneIds': (list(genes.keys()), 'hasAny({value}, {field})')}, {'gt': (None, 'isNull({field})')}, diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index e98752d0d0..71606ab2de 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -10,13 +10,14 @@ from clickhouse_search.backend.fields import NamedTupleField from clickhouse_search.backend.functions import Array, ArrayFilter, ArrayIntersect, ArraySort, GroupArrayArray, If, Tuple, \ ArrayMap, Modulo -from clickhouse_search.managers import InvalidDatasetTypeException +from clickhouse_search.managers import InvalidDatasetTypeException, InvalidSearchException from clickhouse_search.models.gt_stats_models import PROJECT_GT_STATS_VIEW_CLASS_MAP from clickhouse_search.models.reference_data_models import BaseClinvar, BaseHgmd from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, \ ENTRY_CLASS_MAP, VARIANTS_CLASS_MAP, VARIANT_DETAILS_CLASS_MAP from reference_data.models import GeneInfo, GeneConstraint, Omim, GENOME_VERSION_LOOKUP from seqr.models import Sample, PhenotypePrioritization, Family, Individual +from seqr.utils.gene_utils import parse_locus_list_items from seqr.utils.logging_utils import SeqrLogger from clickhouse_search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \ PRIORITIZED_GENE_SORT, COMPOUND_HET, COMPOUND_HET_ALLOW_HOM_ALTS, RECESSIVE, AFFECTED, MALE_SEXES, \ @@ -25,41 +26,45 @@ logger = SeqrLogger(__name__) -class InvalidSearchException(Exception): - pass BATCH_SIZE = 10000 +MAX_GENES_FOR_FILTER = 10000 +MAX_NO_LOCATION_COMP_HET_FAMILIES = 100 +MIN_MULTI_FAMILY_SEQR_AC = 5000 TRANSCRIPT_CONSEQUENCES_FIELD = 'sortedTranscriptConsequences' SELECTED_GENE_FIELD = 'selectedGeneId' SELECTED_TRANSCRIPT_FIELD = 'selectedTranscript' -def get_clickhouse_variants(families, search, user, genome_version, sort=None, sample_data_by_dataset_type=None): - inheritance_mode = search.get('inheritance_mode') +def get_clickhouse_variants(families, user, genome_version=None, sort=None, sample_data_by_dataset_type=None, inheritance=None, locus=None, exclude_keys=None, exclude_key_pairs=None, no_access_project_genome_version=None, **search): + genome_version = genome_version or no_access_project_genome_version or _get_search_genome_version(families) + + search['inheritance_filter'] = (inheritance or {}).get('filter') or {} + inheritance_mode = None if search['inheritance_filter'].get('genotype') else (inheritance or {}).get('mode') + _parse_locus_search(locus or {}, genome_version, search) + has_comp_het = inheritance_mode in {RECESSIVE, COMPOUND_HET} - has_x_chrom_comp_het = has_comp_het and _is_x_chrom_only(genome_version, **search) - has_x_linked = inheritance_mode in {RECESSIVE, X_LINKED_RECESSIVE} and _has_x_chrom(genome_version, **search) + has_x_chrom_comp_het = has_comp_het and _is_x_chrom_only(**search) + has_x_linked = inheritance_mode in {RECESSIVE, X_LINKED_RECESSIVE} and _has_x_chrom(**search) sample_data_by_dataset_type = sample_data_by_dataset_type or {} results = [] - exclude_keys = search.pop('exclude_keys', None) or {} - exclude_key_pairs = search.pop('exclude_key_pairs', None) or {} searched_dataset_types = set() sample_data_errors = set() for dataset_type in ENTRY_CLASS_MAP[genome_version]: try: entry_qs, variants_qs, parsed_filters = _parse_dataset_type_query( genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, - annotate_affected_males=has_x_chrom_comp_het or has_x_linked, **search, + annotate_affected_males=has_x_chrom_comp_het or has_x_linked, + allow_no_samples=bool(no_access_project_genome_version), inheritance_mode=inheritance_mode, **search, ) except InvalidDatasetTypeException: continue - if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and search.get('no_access_project_genome_version'): - logger.info('Looking up variants in projects with no user access', user) - results += _get_search_results( - entry_qs, variants_qs, **search, **parsed_filters, sample_data=None, skip_entry_fields=True, - exclude_projects=sample_data_by_dataset_type[dataset_type].get('project_guids'), + if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and no_access_project_genome_version: + results += _get_no_access_search_results( + entry_qs, variants_qs, has_comp_het, user, **search, **parsed_filters, + exclude_projects=sample_data_by_dataset_type[dataset_type].get('project_guids'), inheritance_mode=inheritance_mode, ) searched_dataset_types.add(dataset_type) @@ -72,26 +77,22 @@ def get_clickhouse_variants(families, search, user, genome_version, sort=None, s dataset_results = [] if inheritance_mode != COMPOUND_HET: dataset_results += _get_search_results( - entry_qs, variants_qs, sample_data, exclude_keys=exclude_keys.get(dataset_type), **search, **parsed_filters, + entry_qs, variants_qs, sample_data, inheritance_mode=inheritance_mode, exclude_keys=(exclude_keys or {}).get(dataset_type), **search, **parsed_filters, ) run_x_linked_male_search = has_x_linked and not (inheritance_mode == X_LINKED_RECESSIVE and sample_data.get('samples')) if run_x_linked_male_search: dataset_results += _get_x_linked_male_search_results( - entry_qs, variants_qs, dataset_type, user, sample_data, exclude_keys=exclude_keys.get(dataset_type), + entry_qs, variants_qs, dataset_type, user, sample_data, exclude_keys=(exclude_keys or {}).get(dataset_type), **search, **parsed_filters, ) if has_comp_het: - comp_het_sample_data = sample_data - if has_x_chrom_comp_het and 'affected_male_family_guids' in sample_data and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: - comp_het_sample_data = _no_affected_male_families(sample_data, user) - result_q = _get_data_type_comp_het_results_queryset( - entry_qs, variants_qs, comp_het_sample_data, exclude_key_pairs=exclude_key_pairs.get(dataset_type), - **search, **parsed_filters, + dataset_results += _get_data_type_comp_het_results_queryset( + entry_qs, variants_qs, sample_data, user, parsed_filters, **search, + exclude_key_pairs=(exclude_key_pairs or {}).get(dataset_type), + is_x_chrom=has_x_chrom_comp_het and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS, ) - if result_q is not None: - dataset_results += _evaluate_results(result_q, is_comp_het=True) if 'samples' not in sample_data: _add_individual_guids(dataset_results) @@ -99,7 +100,7 @@ def get_clickhouse_variants(families, search, user, genome_version, sort=None, s searched_dataset_types.add(dataset_type) if has_comp_het and any(dt.startswith(Sample.DATASET_TYPE_SV_CALLS) for dt in VARIANTS_CLASS_MAP[genome_version]): - results += _get_multi_data_type_comp_het_results(genome_version, families, sample_data_by_dataset_type, user, exclude_key_pairs, searched_dataset_types, **search) + results += _get_multi_data_type_comp_het_results(genome_version, families, sample_data_by_dataset_type, user, exclude_key_pairs or {}, searched_dataset_types, **search) if not searched_dataset_types: _raise_dataset_type_errors(sample_data_errors, sample_data_by_dataset_type) @@ -107,7 +108,56 @@ def get_clickhouse_variants(families, search, user, genome_version, sort=None, s logger.info(f'Total results: {len(results)}', user) return get_sorted_search_results(results, sort, families) -def _parse_dataset_type_query(genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, annotate_affected_males, inheritance_mode=None, inheritance_filter=None, no_access_project_genome_version=None, **search_kwargs): + +def _get_search_genome_version(families): + genome_versions = families.values_list('project__genome_version', flat=True).distinct() + if len (genome_versions) > 1: + project_versions = families.values('project__genome_version').annotate( + projects=ArrayAgg('project__name', distinct=True) + ) + summary = '; '.join( + sorted([f"{agg['project__genome_version']} - {', '.join(sorted(agg['projects']))}" for agg in project_versions])) + raise InvalidSearchException( + f'Searching across multiple genome builds is not supported. Remove projects with differing genome builds from search: {summary}') + + return genome_versions[0] + + +def _parse_locus_search(locus, genome_version, search): + exclude = search.get('exclude') or {} + exclude_locations = bool(exclude.get('rawItems')) + if locus and exclude_locations: + raise InvalidSearchException('Cannot specify both Location and Excluded Genes/Intervals') + + genes, intervals, invalid_items = parse_locus_list_items( + locus or exclude, genome_version=genome_version, get_genes_func=_get_genes, + ) + if invalid_items: + raise InvalidSearchException('Invalid genes/intervals: {}'.format(', '.join(invalid_items))) + if (genes or intervals) and len(genes) + len(intervals) > MAX_GENES_FOR_FILTER: + raise InvalidSearchException('Too many genes/intervals') + + exclude_intervals = None + if exclude_locations: + exclude_intervals = intervals + exclude_intervals += list(genes.values()) + genes = None + intervals = None + + search.update({ + 'genes': genes, 'intervals': intervals, 'exclude_intervals': exclude_intervals, 'raw_variant_items': locus.get('rawVariantItems'), + }) + exclude.pop('rawItems', None) + + +def _get_genes(gene_ids, genome_version): + genes = GeneInfo.objects.filter(**{'gene_id__in': gene_ids, f'start_grch{genome_version}__isnull': False}).values( + 'id', 'gene_id', **{field: F(f'{field}_grch{genome_version}') for field in ['chrom', 'start', 'end']}, + ) + return {gene.pop('gene_id'): gene for gene in genes} + + +def _parse_dataset_type_query(genome_version, dataset_type, families, sample_data_by_dataset_type, sample_data_errors, annotate_affected_males, inheritance_mode=None, inheritance_filter=None, allow_no_samples=False, **search_kwargs): entry_qs = ENTRY_CLASS_MAP[genome_version][dataset_type].objects.filter_locus(inheritance_mode=inheritance_mode, **search_kwargs) variants_qs = VARIANTS_CLASS_MAP[genome_version][dataset_type].objects parsed_filters = variants_qs.get_parsed_annotations_filters(**search_kwargs) @@ -121,7 +171,8 @@ def _parse_dataset_type_query(genome_version, dataset_type, families, sample_dat inheritance_mode=inheritance_mode, inheritance_filter=inheritance_filter, annotate_affected_males=annotate_affected_males, - allow_no_samples=bool(no_access_project_genome_version), + has_location_filter=any(search_kwargs.get(field) for field in ['genes', 'intervals', 'variant_ids']), + allow_no_samples=allow_no_samples, ) sample_data_by_dataset_type[dataset_type] = sample_data @@ -155,7 +206,16 @@ def _get_search_results(entry_qs, variants_qs, sample_data, skip_entry_fields=Fa return _evaluate_results(results.result_values(skip_entry_fields=skip_entry_fields)) -def _get_x_linked_male_search_results(entry_qs, variants_qs, dataset_type, user, sample_data, inheritance_mode=None, **search_kwargs): +def _get_no_access_search_results(entry_qs, variants_qs, has_comp_het, user, **search_kwargs): + if len(search_kwargs.get('genes') or []) != 1: + raise InvalidSearchException('Including external projects is only available when searching for a single gene') + if has_comp_het: + raise InvalidSearchException('Compound heterozygous search is not supported when including external projects') + logger.info('Looking up variants in projects with no user access', user) + return _get_search_results(entry_qs, variants_qs, **search_kwargs, sample_data=None, skip_entry_fields=True) + + +def _get_x_linked_male_search_results(entry_qs, variants_qs, dataset_type, user, sample_data, **search_kwargs): affected_male_family_guids = { s['family_guid'] for s in sample_data['samples'] if s['affected'] == AFFECTED and s['sex'] in MALE_SEXES } if 'samples' in sample_data else sample_data['affected_male_family_guids'] @@ -179,11 +239,11 @@ def _evaluate_results(result_q, is_comp_het=False): raise InvalidSearchException('This search returned too many results') return results -def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_data_by_dataset_type, user, exclude_key_pairs, searched_dataset_types, annotations=None, annotations_secondary=None, inheritance_mode=None, **search_kwargs): +def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_data_by_dataset_type, user, exclude_key_pairs, searched_dataset_types, annotations=None, annotations_secondary=None, **search_kwargs): if annotations_secondary: annotations = { **annotations, - **{k: v + annotations[k] if k in annotations else v for k, v in annotations_secondary.items()}, + **{k: v + annotations[k] if annotations.get(k) else v for k, v in annotations_secondary.items()}, } try: @@ -273,23 +333,30 @@ def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_d return results -def _get_data_type_comp_het_results_queryset(entry_qs, variants_qs, sample_data, annotations_secondary=None, inheritance_mode=None, exclude_key_pairs=None, **search_kwargs): +def _get_data_type_comp_het_results_queryset(entry_qs, variants_qs, sample_data, user, parsed_filters, is_x_chrom=False, annotations_secondary=None, exclude_key_pairs=None, **search_kwargs): + if not any(parsed_filters.values()): + raise InvalidSearchException('Annotations must be specified to search for compound heterozygous variants') + parsed_secondary_filters = {} if annotations_secondary: try: parsed_secondary_filters = variants_qs.get_parsed_annotations_filters(annotations=annotations_secondary) except InvalidDatasetTypeException: - return None + return [] + + if is_x_chrom and 'affected_male_family_guids' in sample_data: + sample_data = _no_affected_male_families(sample_data, user) entries = entry_qs.search( - sample_data, **search_kwargs, inheritance_mode=COMPOUND_HET, annotate_carriers=True, + sample_data, **search_kwargs, **parsed_filters, inheritance_mode=COMPOUND_HET, annotate_carriers=True, ) - primary_q = variants_qs.subquery_join(entries).search(**search_kwargs) - secondary_kwargs = {**search_kwargs, **parsed_secondary_filters} + primary_q = variants_qs.subquery_join(entries).search(**search_kwargs, **parsed_filters) + secondary_kwargs = {**search_kwargs, **parsed_filters, **parsed_secondary_filters} secondary_q = variants_qs.subquery_join(entries).search(**secondary_kwargs) - return _get_comp_het_results_queryset(variants_qs, primary_q, secondary_q, sample_data['num_families'], exclude_key_pairs) + result_q = _get_comp_het_results_queryset(variants_qs, primary_q, secondary_q, sample_data['num_families'], exclude_key_pairs) + return _evaluate_results(result_q, is_comp_het=True) if result_q is not None else [] def _get_comp_het_results_queryset(variants_qs, primary_q, secondary_q, num_families, exclude_key_pairs): @@ -411,8 +478,12 @@ def get_sorted_search_results(results, sort, families): return sorted_results -def format_clickhouse_export_results(results, genome_version): +def format_clickhouse_export_results(results): formatted_results = [variant for result in results for variant in (result if isinstance(result, list) else [result])] + if not formatted_results: + return [] + + genome_version = formatted_results[0]['genomeVersion'] keys_with_no_details = {result['key'] for result in formatted_results if not 'transcripts' in result} detail_qs = get_variant_details_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details) details_by_key = { @@ -439,7 +510,11 @@ def format_clickhouse_export_results(results, genome_version): return formatted_results -def format_clickhouse_results(results, genome_version): +def format_clickhouse_results(results): + if not results: + return [] + + genome_version = (results[0] if isinstance(results[0], list) else results)[0]['genomeVersion'] keys_with_no_details = { variant['key'] for result in results for variant in (result if isinstance(result, list) else [result]) if not 'transcripts' in variant } @@ -548,7 +623,7 @@ def _get_sample_metadata(samples, affected_family_only, annotate_affected_males) return samples.aggregate(**annotations) -def _get_sample_data(families, dataset_type, annotate_affected_males=False, allow_no_samples=False, inheritance_mode=None, inheritance_filter=None): +def _get_sample_data(families, dataset_type, annotate_affected_males=False, allow_no_samples=False, inheritance_mode=None, inheritance_filter=None, has_location_filter=False): sample_type = None if dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS): dataset_type, sample_type = dataset_type.split('_') @@ -561,6 +636,14 @@ def _get_sample_data(families, dataset_type, annotate_affected_males=False, allo sample_data = _get_sample_metadata(samples, affected_family_only, annotate_affected_males) family_guids = set(sample_data.pop('family_guids')) + if not has_location_filter: + if len(sample_data['project_guids']) > 1: + raise InvalidSearchException('Location must be specified to search across multiple projects') + if inheritance_mode in {RECESSIVE, COMPOUND_HET} and len(family_guids) > MAX_NO_LOCATION_COMP_HET_FAMILIES: + raise InvalidSearchException( + 'Location must be specified to search for compound heterozygous variants across many families', + ) + sample_data['num_families'] = len(family_guids) if sample_type: sample_data['sample_type_families'] = {sample_type: family_guids} @@ -639,16 +722,26 @@ def _affected_male_families(sample_data, affected_male_family_guids): } -def _is_x_chrom_only(genome_version, genes=None, intervals=None, **kwargs): +def _is_x_chrom_only(genes=None, intervals=None, **kwargs): if not (genes or intervals): return False - return all('X' in gene[f'chromGrch{genome_version}'] for gene in (genes or {}).values()) and all('X' in interval['chrom'] for interval in (intervals or [])) + return all('X' in interval['chrom'] for interval in list((genes or {}).values()) + (intervals or [])) -def _has_x_chrom(genome_version, genes=None, intervals=None, **kwargs): +def _has_x_chrom(genes=None, intervals=None, **kwargs): if not (genes or intervals): return True - return any('X' in gene[f'chromGrch{genome_version}'] for gene in (genes or {}).values()) or any('X' in interval['chrom'] for interval in (intervals or [])) + return any('X' in interval['chrom'] for interval in list((genes or {}).values()) + (intervals or [])) + + +def _prioritized_gene_sort(gene_ids, families): + if len(families) != 1: + raise InvalidSearchException('Phenotype sort is only supported for single-family search.') + return { + agg['gene_id']: agg['min_rank'] for agg in PhenotypePrioritization.objects.filter( + gene_id__in=gene_ids, individual__family_id=families[0].id, rank__lte=100, + ).values('gene_id').annotate(min_rank=Min('rank')) + } OMIM_SORT = 'in_omim' @@ -660,11 +753,7 @@ def _has_x_chrom(genome_version, genes=None, intervals=None, **kwargs): OMIM_SORT: lambda gene_ids, _: set(Omim.objects.filter( gene__gene_id__in=gene_ids, phenotype_mim_number__isnull=False, ).values_list('gene__gene_id', flat=True)), - PRIORITIZED_GENE_SORT: lambda gene_ids, families: { - agg['gene_id']: agg['min_rank'] for agg in PhenotypePrioritization.objects.filter( - gene_id__in=gene_ids, individual__family_id=families[0].id, rank__lte=100, - ).values('gene_id').annotate(min_rank=Min('rank')) - }, + PRIORITIZED_GENE_SORT: _prioritized_gene_sort, } def _get_sort_gene_metadata(sort, results, families): @@ -796,7 +885,7 @@ def _clickhouse_variant_lookup(entries, genome_version, data_type, affected_only variant = results.result_values().first() if variant: - variant = format_clickhouse_results([variant], genome_version)[0] + variant = format_clickhouse_results([variant])[0] return variant def _filter_lookup_entries(entries, affected_only, hom_only): @@ -806,12 +895,12 @@ def _filter_lookup_entries(entries, affected_only, hom_only): entries = entries.filter(calls__array_exists={'gt': (2,)}) return entries -def clickhouse_variant_lookup(user, variant_id, parsed_variant_id, sample_type, genome_version, affected_only, hom_only): +def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, affected_only, hom_only): data_type = entry_qs = None for dataset_type, entry_cls in sorted(ENTRY_CLASS_MAP[genome_version].items()): try: entry_qs = entry_cls.objects.filter_locus( - variant_ids=[variant_id], parsed_variant_ids=[parsed_variant_id] if parsed_variant_id else [], + variant_ids=[variant_id], raw_variant_items=variant_id, ) except InvalidDatasetTypeException: continue @@ -831,16 +920,17 @@ def clickhouse_variant_lookup(user, variant_id, parsed_variant_id, sample_type, entry_qs, genome_version, data_type, affected_only=affected_only, hom_only=hom_only, ) if variant: - _add_liftover_genotypes(variant, data_type, parsed_variant_id, affected_only, hom_only) + _add_liftover_genotypes(variant, data_type, affected_only, hom_only) else: lifted_genome_version = next(gv for gv in ENTRY_CLASS_MAP.keys() if gv != genome_version) lifted_entry_cls = ENTRY_CLASS_MAP[lifted_genome_version].get(data_type) if lifted_entry_cls: from seqr.utils.search.utils import run_liftover - liftover_results = run_liftover(lifted_genome_version, parsed_variant_id[0], parsed_variant_id[1]) + chrom, pos, base_id = variant_id.split('-', 2) + liftover_results = run_liftover(lifted_genome_version, chrom, int(pos)) if liftover_results: - lifted_id = (liftover_results[0], liftover_results[1], *parsed_variant_id[2:]) - entry_qs = lifted_entry_cls.objects.filter_locus(parsed_variant_ids=[lifted_id]) + lifted_id = f'{liftover_results[0]}-{liftover_results[1]}-{base_id}' + entry_qs = lifted_entry_cls.objects.filter_locus(raw_variant_items=lifted_id) variant = _clickhouse_variant_lookup( entry_qs, lifted_genome_version, data_type, affected_only=affected_only, hom_only=hom_only, ) @@ -870,12 +960,12 @@ def clickhouse_variant_lookup(user, variant_id, parsed_variant_id, sample_type, return variants -def _add_liftover_genotypes(variant, data_type, variant_id, affected_only, hom_only): +def _add_liftover_genotypes(variant, data_type, affected_only, hom_only): lifted_entry_cls = ENTRY_CLASS_MAP.get(variant.get('liftedOverGenomeVersion'), {}).get(data_type) if not (lifted_entry_cls and variant.get('liftedOverChrom') and variant.get('liftedOverPos')): return - lifted_id = (variant['liftedOverChrom'], variant['liftedOverPos'], *variant_id[2:]) - lifted_entries = lifted_entry_cls.objects.filter_locus(parsed_variant_ids=[lifted_id]) + lifted_id = f"{variant['liftedOverChrom']}-{variant['liftedOverPos']}-{variant['ref']}-{variant['alt']}" + lifted_entries = lifted_entry_cls.objects.filter_locus(raw_variant_items=lifted_id) lifted_entries = _filter_lookup_entries(lifted_entries, affected_only, hom_only) gt_field, gt_expr = lifted_entry_cls.objects.genotype_expression() lifted_entry_data = lifted_entries.values('key').annotate(**{gt_field: GroupArrayArray(gt_expr)}) diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py index a5078a8996..6b8188e078 100644 --- a/clickhouse_search/search_tests.py +++ b/clickhouse_search/search_tests.py @@ -847,8 +847,8 @@ def test_variant_id_search(self): ) @mock.patch('seqr.utils.search.utils.MAX_EXPORT_VARIANTS', 2) - @mock.patch('seqr.utils.search.utils.MAX_GENES_FOR_FILTER', 2) - @mock.patch('seqr.utils.search.utils.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) + @mock.patch('clickhouse_search.search.MAX_GENES_FOR_FILTER', 2) + @mock.patch('clickhouse_search.search.MAX_NO_LOCATION_COMP_HET_FAMILIES', 1) @mock.patch('clickhouse_search.search.MAX_VARIANTS', 3) def test_invalid_search(self): url = reverse(query_variants_handler, args=['not_a_hash']) @@ -861,7 +861,7 @@ def test_invalid_search(self): self._assert_expected_search_error('Invalid search: no projects/ families specified', project_families=[]) search_hash = self._assert_expected_search_error( - 'Invalid variants: chr2-A-C', locus={'rawVariantItems': 'chr2-A-C'}, + 'Invalid variants: 2-A-C', locus={'rawVariantItems': 'chr2-A-C'}, ) self.set_cache(f'search_results__VRS{search_hash:07d}__xpos', [VARIANT1, VARIANT2, VARIANT3, VARIANT4]) @@ -896,12 +896,14 @@ def test_invalid_search(self): pathogenicity={'clinvar': ['pathogenic', 'vus']}, exclude={'clinvar': ['benign', 'vus']}, ) - self._assert_expected_search_error('Annotations must be specified to search for compound heterozygous variants', inheritance_mode='recessive') - - annotations = {'frameshift': ['frameshift_variant']} self._assert_expected_search_error( 'Location must be specified to search for compound heterozygous variants across many families', - annotations=annotations, inheritance_mode='recessive', + inheritance_mode='recessive', + ) + + self._assert_expected_search_error( + 'Annotations must be specified to search for compound heterozygous variants', + inheritance_mode='recessive', locus={'rawItems': GENE_IDS[0]}, ) self._assert_expected_search_error( @@ -911,6 +913,7 @@ def test_invalid_search(self): self._assert_expected_search_error('seqr AC frequency of at least 5000 must be specified to search across multiple families', freqs={}) + annotations = {'frameshift': ['frameshift_variant']} self._assert_expected_search_error( 'Inheritance based search is disabled in families with no data loaded for affected individuals', inheritance_mode='recessive', project_families=[{'projectGuid': 'R0001_1kg', 'familyGuids': ['F000005_5']}], diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py index 3abf1f277e..9e11b9296a 100644 --- a/seqr/management/commands/tag_seqr_prioritized_variants.py +++ b/seqr/management/commands/tag_seqr_prioritized_variants.py @@ -164,7 +164,7 @@ } NO_PANEL_APP_DE_NOVO_SEARCH = { - 'inheritance_mode': DE_NOVO, + 'inheritance': {'mode': DE_NOVO}, 'freqs': { 'callset': {'ac': 100}, 'gnomad_exomes': {'ac': 100}, @@ -183,7 +183,7 @@ 'SNV_INDEL': { 'Clinvar Pathogenic': { 'gene_list_moi': DOMINANT_MOI, - 'inheritance_mode': ANY_AFFECTED, + 'inheritance': {'mode': ANY_AFFECTED}, 'pathogenicity': CLINVAR_FILTER, 'freqs': { 'callset': {'ac': 150}, @@ -192,28 +192,28 @@ }, }, 'Clinvar Pathogenic - Compound Heterozygous': { - 'inheritance_mode': COMPOUND_HET, + 'inheritance': {'mode': COMPOUND_HET}, 'annotations': {}, 'annotations_secondary': HIGH_MODERATE_ANNOTATIONS, **CLINVAR_RECESSIVE_SEARCH, }, 'Clinvar Both Pathogenic - Compound Heterozygous': { - 'inheritance_mode': COMPOUND_HET, + 'inheritance': {'mode': COMPOUND_HET}, **CLINVAR_RECESSIVE_SEARCH, }, 'Clinvar Pathogenic - Recessive': { - 'inheritance_mode': HOMOZYGOUS_RECESSIVE, + 'inheritance': {'mode': HOMOZYGOUS_RECESSIVE}, **CLINVAR_RECESSIVE_SEARCH, }, 'Clinvar Pathogenic - X-Linked Recessive': { - 'inheritance_mode': X_LINKED_RECESSIVE_MALE_AFFECTED, + 'inheritance': {'mode': X_LINKED_RECESSIVE_MALE_AFFECTED}, 'family_filter': { AFFECTED_MALE_FAMILY_FILTER: True }, **CLINVAR_RECESSIVE_SEARCH, }, 'Compound Heterozygous': { - 'inheritance_mode': COMPOUND_HET, + 'inheritance': {'mode': COMPOUND_HET}, 'annotations': HIGH_ANNOTATIONS, 'annotations_secondary': HIGH_MODERATE_ANNOTATIONS, **RECESSIVE_SEARCH, @@ -222,22 +222,31 @@ 'family_filter': { CONFIRMED_FAMILY_FILTER: True }, - 'inheritance_mode': COMPOUND_HET, + 'inheritance': {'mode': COMPOUND_HET}, 'annotations': MODERATE_ANNOTATIONS, **RECESSIVE_SEARCH, }, 'Compound Heterozygous - Both High Splice AI': { - 'inheritance_mode': COMPOUND_HET, - **HIGH_SPLICE_AI_SEARCH, + 'family_filter': { + CONFIRMED_FAMILY_FILTER: False + }, + 'inheritance': {'mode': COMPOUND_HET}, + 'annotations':{ + 'splice_ai': 0.8, + }, **RECESSIVE_SEARCH_NO_IN_SILICO, }, 'Compound Heterozygous - Both High Splice AI - Confirmed': { - 'inheritance_mode': COMPOUND_HET, - **CONFIRMED_HIGH_SPLICE_AI_SEARCH, + 'family_filter': { + CONFIRMED_FAMILY_FILTER: True + }, + 'annotations': { + 'splice_ai': 0.5, + }, **RECESSIVE_SEARCH_NO_IN_SILICO, }, 'Compound Heterozygous - Clinvar Pathogenic/ High Splice AI': { - 'inheritance_mode': COMPOUND_HET, + 'inheritance': {'mode': COMPOUND_HET}, 'annotations': {}, 'annotations_secondary': { 'splice_ai': 0.5, @@ -248,8 +257,7 @@ 'family_filter': { CONFIRMED_FAMILY_FILTER: False }, - 'inheritance_mode': COMPOUND_HET, - 'no_secondary_annotations': True, + 'inheritance': {'mode': COMPOUND_HET}, 'annotations': HIGH_MODERATE_ANNOTATIONS, 'annotations_secondary':{ 'splice_ai': 0.8, @@ -264,8 +272,7 @@ 'family_filter': { CONFIRMED_FAMILY_FILTER: True }, - 'inheritance_mode': COMPOUND_HET, - 'no_secondary_annotations': True, + 'inheritance': {'mode': COMPOUND_HET}, 'annotations': HIGH_MODERATE_ANNOTATIONS, 'annotations_secondary':{ 'splice_ai': 0.5, @@ -314,17 +321,17 @@ **NO_PANEL_APP_DE_NOVO_SEARCH, }, 'High Splice AI - Recessive': { - 'inheritance_mode': HOMOZYGOUS_RECESSIVE, + 'inheritance': {'mode': HOMOZYGOUS_RECESSIVE}, **HIGH_SPLICE_AI_SEARCH, **RECESSIVE_SEARCH_NO_IN_SILICO, }, 'High Splice AI - Recessive Confirmed': { - 'inheritance_mode': HOMOZYGOUS_RECESSIVE, + 'inheritance': {'mode': HOMOZYGOUS_RECESSIVE}, **CONFIRMED_HIGH_SPLICE_AI_SEARCH, **RECESSIVE_SEARCH_NO_IN_SILICO, }, 'High Splice AI - X-Linked Recessive': { - 'inheritance_mode': X_LINKED_RECESSIVE_MALE_AFFECTED, + 'inheritance': {'mode': X_LINKED_RECESSIVE_MALE_AFFECTED}, **HIGH_SPLICE_AI_SEARCH, **RECESSIVE_SEARCH_NO_IN_SILICO, 'family_filter': { @@ -333,7 +340,7 @@ }, }, 'High Splice AI - X-Linked Recessive Confirmed': { - 'inheritance_mode': X_LINKED_RECESSIVE_MALE_AFFECTED, + 'inheritance': {'mode': X_LINKED_RECESSIVE_MALE_AFFECTED}, **CONFIRMED_HIGH_SPLICE_AI_SEARCH, **RECESSIVE_SEARCH_NO_IN_SILICO, 'family_filter': { @@ -342,12 +349,12 @@ }, }, 'Recessive': { - 'inheritance_mode': HOMOZYGOUS_RECESSIVE, + 'inheritance': {'mode': HOMOZYGOUS_RECESSIVE}, 'annotations': HIGH_MODERATE_ANNOTATIONS, **RECESSIVE_SEARCH, }, 'X-Linked Recessive': { - 'inheritance_mode': X_LINKED_RECESSIVE_MALE_AFFECTED, + 'inheritance': {'mode': X_LINKED_RECESSIVE_MALE_AFFECTED}, 'family_filter': { AFFECTED_MALE_FAMILY_FILTER: True }, @@ -360,7 +367,7 @@ 'family_filter': { CONFIRMED_FAMILY_FILTER: True }, - 'inheritance_mode': COMPOUND_HET, + 'inheritance': {'mode': COMPOUND_HET}, **SV_RECESSIVE_SEARCH, 'qualityFilter': PASS_QUALITY_FILTER, }, @@ -375,11 +382,11 @@ 'family_filter': { CONFIRMED_FAMILY_FILTER: True }, - 'inheritance_mode': HOMOZYGOUS_RECESSIVE, + 'inheritance': {'mode': HOMOZYGOUS_RECESSIVE}, **SV_RECESSIVE_SEARCH, }, 'SV - X-Linked Recessive': { - 'inheritance_mode': X_LINKED_RECESSIVE_MALE_AFFECTED, + 'inheritance': {'mode': X_LINKED_RECESSIVE_MALE_AFFECTED}, 'family_filter': { AFFECTED_MALE_FAMILY_FILTER: True, CONFIRMED_FAMILY_FILTER: True, @@ -389,20 +396,22 @@ }, 'MITO': { 'Mitochondrial - Pathogenic': { - 'inheritance_mode': ANY_AFFECTED, + 'inheritance': {'mode': ANY_AFFECTED}, 'pathogenicity': {'clinvar': CLINVAR_FILTER['clinvar']}, 'annotations': { 'mitomap_pathogenic': True, }, 'freqs': { + 'callset': {'ac': 5000}, 'gnomad_mito': {'af': 0.05}, }, }, 'Mitochondrial - De Novo/ Dominant': { 'gene_list_moi': MITO_MOI, - 'inheritance_mode': DE_NOVO, + 'inheritance': {'mode': DE_NOVO}, 'annotations': HIGH_MODERATE_ANNOTATIONS, 'freqs': { + 'callset': {'ac': 5000}, 'gnomad_mito': {'af': 0.001}, }, 'in_silico': { @@ -621,7 +630,7 @@ def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by } num_results = cls._execute_search( sample_data_by_dataset_type, search_name, family_variant_data, family_guid_map, - inheritance_mode=COMPOUND_HET, **config_search, **ALL_SEARCHES_CRITERIA, genes=genes, + inheritance={'mode': COMPOUND_HET}, **config_search, **ALL_SEARCHES_CRITERIA, genes=genes, ) search_counts[search_name] = num_results @@ -630,10 +639,10 @@ def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by @staticmethod def _execute_search(sample_data_by_dataset_type, search_name, family_variant_data, family_guid_map, **kwargs): results = get_clickhouse_variants( - families=None, search=kwargs, user=None, genome_version=GENOME_VERSION_GRCh38, + families=family_guid_map.keys(), user=None, genome_version=GENOME_VERSION_GRCh38, sample_data_by_dataset_type={ **{dt: None for dt in ENTRY_CLASS_MAP[GENOME_VERSION_GRCh38]}, **sample_data_by_dataset_type, - }, + }, **kwargs ) for result in results: if isinstance(result, list): diff --git a/seqr/utils/gene_utils.py b/seqr/utils/gene_utils.py index 1b2b615715..f7b6152147 100644 --- a/seqr/utils/gene_utils.py +++ b/seqr/utils/gene_utils.py @@ -158,7 +158,7 @@ def _process_result(result, gene): return _get_json_for_models(genes, process_result=_process_result, **kwargs) -def parse_locus_list_items(request_json, genome_version=None, **kwargs): +def parse_locus_list_items(request_json, genome_version=None, get_genes_func=get_genes): raw_items = request_json.get('rawItems') if not raw_items: return None, None, None @@ -196,6 +196,6 @@ def parse_locus_list_items(request_json, genome_version=None, **kwargs): gene_symbols_to_ids = get_gene_ids_for_gene_symbols(gene_symbols, genome_version=genome_version) invalid_items += [symbol for symbol in gene_symbols if not gene_symbols_to_ids.get(symbol)] gene_ids.update({gene_ids[0] for gene_ids in gene_symbols_to_ids.values() if len(gene_ids)}) - genes_by_id = get_genes(list(gene_ids), genome_version=genome_version, **kwargs) if gene_ids else {} + genes_by_id = get_genes_func(list(gene_ids), genome_version=genome_version) if gene_ids else {} invalid_items += [gene_id for gene_id in gene_ids if not genes_by_id.get(gene_id)] return genes_by_id, intervals, invalid_items \ No newline at end of file diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 2b52f3fe96..8df369fd04 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -7,52 +7,24 @@ from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \ get_sorted_search_results, clickhouse_variant_lookup, InvalidSearchException from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37 -from seqr.models import Sample, Project, VariantSearchResults +from seqr.models import Sample, VariantSearchResults from seqr.utils.logging_utils import SeqrLogger from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_get_wildcard_json, safe_redis_set_json -from clickhouse_search.constants import XPOS_SORT_KEY, PRIORITIZED_GENE_SORT, RECESSIVE, COMPOUND_HET, \ - PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY -from seqr.utils.gene_utils import parse_locus_list_items -from seqr.utils.xpos_utils import get_xpos, format_chrom +from clickhouse_search.constants import XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY +from seqr.utils.xpos_utils import parse_variant_id from seqr.views.utils.permissions_utils import user_is_analyst logger = SeqrLogger(__name__) - -MAX_GENES_FOR_FILTER = 10000 -MIN_MULTI_FAMILY_SEQR_AC = 5000 MAX_EXPORT_VARIANTS = 1000 -MAX_NO_LOCATION_COMP_HET_FAMILIES = 100 - - -def _get_search_genome_version(search_model): - projects = Project.objects.filter(family__in=search_model.families.all()).values_list('genome_version', 'name').distinct() - project_versions = defaultdict(set) - for genome_version, project_name in projects: - project_versions[genome_version].add(project_name) - - if len(project_versions) > 1: - summary = '; '.join( - [f"{build} - {', '.join(sorted(projects))}" for build, projects in sorted(project_versions.items())]) - raise InvalidSearchException( - f'Searching across multiple genome builds is not supported. Remove projects with differing genome builds from search: {summary}') - if not project_versions: - return search_model.variant_search.search.get('no_access_project_genome_version') - return next(iter(project_versions.keys())) - - -def get_single_variant(family, variant_id, user): - parsed_variant_id = parse_variant_id(variant_id) - parsed_variant_ids = [parsed_variant_id] if parsed_variant_id else None - search = {'parsed_variant_ids': parsed_variant_ids, 'variant_ids': [variant_id]} - genome_version = family.project.genome_version - variants = get_clickhouse_variants([family], search, user, genome_version) +def get_single_variant(families, variant_id, user): + variants = get_clickhouse_variants(families, user, raw_variant_items=variant_id, variant_ids=[variant_id]) if not variants: raise InvalidSearchException('Variant {} not found'.format(variant_id)) - return format_clickhouse_results(variants, genome_version)[0] + return format_clickhouse_results(variants)[0] def variant_lookup(user, variant_id, genome_version, sample_type=None, affected_only=False, hom_only=False): @@ -66,8 +38,7 @@ def variant_lookup(user, variant_id, genome_version, sample_type=None, affected_ if variants: return variants - parsed_variant_id = parse_variant_id(variant_id) - variants = clickhouse_variant_lookup(user, variant_id, parsed_variant_id, sample_type, genome_version, affected_only, hom_only) + variants = clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, affected_only, hom_only) safe_redis_set_json(cache_key, variants, expire=timedelta(weeks=2)) return variants @@ -78,11 +49,11 @@ def _get_search_cache_key(search_model, sort=None): def export_variants(search_model, user): - search_results, genome_version = _query_variants(search_model, user) + search_results = _query_variants(search_model, user) total_variants = len(search_results) if total_variants > MAX_EXPORT_VARIANTS: raise InvalidSearchException(f'Unable to export more than {MAX_EXPORT_VARIANTS} variants ({total_variants} requested)') - return format_clickhouse_export_results(search_results, genome_version) + return format_clickhouse_export_results(search_results) def _get_previous_search_results(search_model, sort): @@ -105,81 +76,37 @@ def _get_previous_search_results(search_model, sort): def query_variants(search_model, sort, page, num_results, user): if sort == PATHOGENICTY_SORT_KEY and user_is_analyst(user): sort = PATHOGENICTY_HGMD_SORT_KEY - all_results, genome_version = _query_variants(search_model, user, sort=sort or XPOS_SORT_KEY) + all_results = _query_variants(search_model, user, sort=sort or XPOS_SORT_KEY) - results_page = format_clickhouse_results(all_results[(page-1)*num_results:page*num_results], genome_version) + results_page = format_clickhouse_results(all_results[(page-1)*num_results:page*num_results]) return results_page, len(all_results) def _query_variants(search_model, user, sort=None): - genome_version = _get_search_genome_version(search_model) previous_search_results = _get_previous_search_results(search_model, sort) or {} if previous_search_results: - return previous_search_results, genome_version + return previous_search_results search = deepcopy(search_model.variant_search.search) families = search_model.families.all() - _validate_sort(sort, families) - parsed_search = _parse_search(search, genome_version, user) - _validate_search(parsed_search, families) + exclude_search = {} + exclude_previous_hash = (search.get('exclude') or {}).pop('previousSearchHash', None) + if exclude_previous_hash: + exclude_search = _get_clickhouse_exclude_keys(exclude_previous_hash, user) - results = get_clickhouse_variants(families, parsed_search, user, genome_version, sort=sort) + results = get_clickhouse_variants(families, user, sort=sort, **search, **exclude_search) cache_key = _get_search_cache_key(search_model, sort=sort) safe_redis_set_json(cache_key, results, expire=timedelta(weeks=2)) - return results, genome_version - - -def _parse_search(search, genome_version, user): - no_access_project_genome_version = search.get('no_access_project_genome_version') - locus = search.pop('locus', None) or {} - exclude = search.get('exclude', None) or {} - exclude_locations = bool(exclude.get('rawItems')) - if locus and exclude_locations: - raise InvalidSearchException('Cannot specify both Location and Excluded Genes/Intervals') - - parsed_search = {**search} - genes, intervals, invalid_items = parse_locus_list_items(locus or exclude, genome_version=genome_version, additional_model_fields=['id']) - if invalid_items: - raise InvalidSearchException('Invalid genes/intervals: {}'.format(', '.join(invalid_items))) - if no_access_project_genome_version and len(genes or []) != 1: - raise InvalidSearchException('Including external projects is only available when searching for a single gene') - if (genes or intervals) and len(genes) + len(intervals) > MAX_GENES_FOR_FILTER: - raise InvalidSearchException('Too many genes/intervals') - parsed_search.update({'genes': genes, 'intervals': intervals, 'exclude_locations': exclude_locations}) - if not (genes or intervals): - variant_ids, parsed_variant_ids, invalid_items = _parse_variant_items(locus) - if invalid_items: - raise InvalidSearchException('Invalid variants: {}'.format(', '.join(invalid_items))) - parsed_search.update({'variant_ids': variant_ids, 'parsed_variant_ids': parsed_variant_ids}) - - exclude.pop('rawItems', None) - if exclude.get('clinvar') and (search.get('pathogenicity') or {}).get('clinvar'): - duplicates = set(search['pathogenicity']['clinvar']).intersection(exclude['clinvar']) - if duplicates: - raise InvalidSearchException(f'ClinVar pathogenicity {", ".join(sorted(duplicates))} is both included and excluded') - - exclude.pop('previousSearch', None) - exclude_previous_hash = exclude.pop('previousSearchHash', None) - if exclude_previous_hash: - parsed_search.update(_get_clickhouse_exclude_keys(exclude_previous_hash, user)) - - for annotation_key in ['annotations', 'annotations_secondary']: - if parsed_search.get(annotation_key): - parsed_search[annotation_key] = {k: v for k, v in parsed_search[annotation_key].items() if v} - - if parsed_search.get('inheritance'): - _parse_inheritance(parsed_search) - - return parsed_search + return results def _get_clickhouse_exclude_keys(search_hash, user): previous_search_model = VariantSearchResults.objects.get(search_hash=search_hash) - results, _ = _query_variants(previous_search_model, user) + results = _query_variants(previous_search_model, user) exclude_keys = defaultdict(list) exclude_key_pairs = defaultdict(list) for variant in results: @@ -202,7 +129,7 @@ def variant_dataset_type(variant): def get_variant_query_gene_counts(search_model, user): - results, _ = _query_variants(search_model, user) + results = _query_variants(search_model, user) flat_variants = [ v for variants in results for v in (variants if isinstance(variants, list) else [variants]) ] @@ -216,83 +143,6 @@ def get_variant_query_gene_counts(search_model, user): return gene_aggs -def _parse_variant_items(search_json): - raw_items = search_json.get('rawVariantItems') - if not raw_items: - return None, None, None - - invalid_items = [] - variant_ids = [] - parsed_variant_ids = [] - for item in raw_items.replace(',', ' ').split(): - variant_id = item.lstrip('chr') - parsed_variant_id = parse_variant_id(variant_id) - if parsed_variant_id: - parsed_variant_ids.append(parsed_variant_id) - variant_ids.append(variant_id) - else: - invalid_items.append(item) - - return variant_ids, parsed_variant_ids, invalid_items - - -def parse_variant_id(variant_id): - try: - return _parse_valid_variant_id(variant_id) - except (KeyError, ValueError): - return None - - -def _parse_valid_variant_id(variant_id): - chrom, pos, ref, alt = variant_id.split('-') - chrom = format_chrom(chrom) - pos = int(pos) - get_xpos(chrom, pos) - return chrom, pos, ref, alt - - -def _validate_sort(sort, families): - if sort == PRIORITIZED_GENE_SORT and len(families) != 1: - raise InvalidSearchException('Phenotype sort is only supported for single-family search.') - - -def _parse_inheritance(search): - inheritance = search.pop('inheritance') - inheritance_mode = inheritance.get('mode') - inheritance_filter = inheritance.get('filter') or {} - - if inheritance_filter.get('genotype'): - inheritance_mode = None - - search.update({'inheritance_mode': inheritance_mode, 'inheritance_filter': inheritance_filter}) - - if not inheritance_mode and list(inheritance_filter.keys()) == ['affected']: - raise InvalidSearchException('Inheritance must be specified if custom affected status is set') - - -def _validate_search(search, families): - has_comp_het_search = search.get('inheritance_mode') in {RECESSIVE, COMPOUND_HET} - has_location_filter = any(search.get(field) for field in ['genes', 'intervals', 'variant_ids']) - if has_comp_het_search: - if not search.get('annotations'): - raise InvalidSearchException('Annotations must be specified to search for compound heterozygous variants') - - if not has_location_filter and len(families) > MAX_NO_LOCATION_COMP_HET_FAMILIES: - raise InvalidSearchException( - 'Location must be specified to search for compound heterozygous variants across many families') - - if search.get('no_access_project_genome_version'): - raise InvalidSearchException('Compound heterozygous search is not supported when including external projects') - - if not has_location_filter and families.values('project_id').distinct().count() > 1: - raise InvalidSearchException('Location must be specified to search across multiple projects') - seqr_ac_filter = search.get('freqs', {}).get('callset', {}).get('ac') or (MIN_MULTI_FAMILY_SEQR_AC + 1) - if seqr_ac_filter > MIN_MULTI_FAMILY_SEQR_AC and len(families) > 1: - raise InvalidSearchException( - f'seqr AC frequency of at least {MIN_MULTI_FAMILY_SEQR_AC} must be specified to search across multiple families' - ) - - LIFTOVERS = { GENOME_VERSION_GRCh38: None, GENOME_VERSION_GRCh37: None, diff --git a/seqr/utils/xpos_utils.py b/seqr/utils/xpos_utils.py index dcab3b78c1..6d06eac680 100644 --- a/seqr/utils/xpos_utils.py +++ b/seqr/utils/xpos_utils.py @@ -80,3 +80,14 @@ def get_chrom_pos(xpos): CHROM_NUMBER_TO_CHROM[chrom_idx], xpos % int(1e9) ) + + +def parse_variant_id(variant_id): + try: + chrom, pos, ref, alt = variant_id.split('-') + chrom = format_chrom(chrom) + pos = int(pos) + get_xpos(chrom, pos) + return chrom, pos, ref, alt + except (KeyError, ValueError): + return None diff --git a/seqr/views/apis/feature_updates_api.py b/seqr/views/apis/feature_updates_api.py index 2fccead30e..140b650182 100644 --- a/seqr/views/apis/feature_updates_api.py +++ b/seqr/views/apis/feature_updates_api.py @@ -11,18 +11,20 @@ TIMEOUT = 5 +def fetch_feature_updates(): + response = requests.get(FEED_URL, timeout=TIMEOUT) + response.raise_for_status() + feed = parse(response.content) + return feed.entries + + def get_feature_updates(request): """ Fetches the feature-updates GitHub Discussion Atom feed, converts feed entries into markdown, and returns markdown and information for each feed entry. """ - response = requests.get(FEED_URL, timeout=TIMEOUT) - response.raise_for_status() - - feed = parse(response.content) - entries = [] - for entry in feed.entries: + for entry in fetch_feature_updates(): # Atom feeds can have multiple content elements per feed entry markdown = "".join(md(content.value) for content in entry.content) entries.append( diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py index e0c62e77d8..09f4ccc04c 100644 --- a/seqr/views/apis/variant_search_api.py +++ b/seqr/views/apis/variant_search_api.py @@ -15,11 +15,12 @@ from reference_data.models import GENOME_VERSION_GRCh38 from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Sample from seqr.utils.search.utils import query_variants, get_single_variant, get_variant_query_gene_counts, \ - variant_lookup, parse_variant_id, export_variants + variant_lookup, export_variants from seqr.utils.search.utils import InvalidSearchException from seqr.views.utils.export_utils import export_table from seqr.utils.gene_utils import get_genes_for_variant_display from seqr.utils.logging_utils import SeqrLogger +from seqr.utils.xpos_utils import parse_variant_id from seqr.views.utils.json_utils import create_json_response, _to_snake_case from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \ create_model_from_json @@ -138,7 +139,7 @@ def query_single_variant_handler(request, variant_id): family = families.first() check_project_permissions(family.project, request.user) - variant = get_single_variant(family, variant_id, request.user) + variant = get_single_variant(families, variant_id, request.user) response = _process_variants([variant], families, request, add_all_context=True, add_locus_list_detail=True) diff --git a/seqr/views/react_app.py b/seqr/views/react_app.py index 0b40591a39..23c9992f44 100644 --- a/seqr/views/react_app.py +++ b/seqr/views/react_app.py @@ -5,6 +5,8 @@ from django.middleware.csrf import rotate_token from django.template import loader from django.http import HttpResponse +import logging + from settings import ( SEQR_VERSION, CSRF_COOKIE_NAME, @@ -15,9 +17,13 @@ VLM_CLIENT_ID, ) from seqr.models import WarningMessage +from seqr.utils.redis_utils import safe_redis_get_json, safe_redis_set_json +from seqr.views.apis.feature_updates_api import fetch_feature_updates from seqr.views.utils.orm_to_json_utils import get_json_for_user, get_json_for_current_user from seqr.views.utils.permissions_utils import login_active_required +logger = logging.getLogger(__name__) + @login_active_required(login_url=LOGIN_URL) def main_app(request, *args, **kwargs): @@ -56,6 +62,7 @@ def render_app_html(request, additional_json=None, include_user=True, status=200 'oauthLoginProvider': SOCIAL_AUTH_PROVIDER, 'vlmEnabled': bool(VLM_CLIENT_ID), 'warningMessages': [message.json() for message in WarningMessage.objects.all()], + 'lastFeatureUpdate': _get_latest_feature_update_date(), }} if include_user: initial_json['user'] = get_json_for_current_user(request.user) @@ -89,3 +96,18 @@ def render_app_html(request, additional_json=None, include_user=True, status=200 ) return HttpResponse(html, content_type="text/html", status=status) + + +FEATURE_CACHE_KEY = 'feature_updates_latest_date' + +def _get_latest_feature_update_date(): + update_date = safe_redis_get_json(FEATURE_CACHE_KEY) + if not update_date: + try: + entries = fetch_feature_updates() + except Exception as e: + logger.error(f'Unable to fetch feature update date: {e}') + return None + update_date = entries[0].published + safe_redis_set_json(FEATURE_CACHE_KEY, update_date, expire=60*60*3) + return update_date diff --git a/seqr/views/react_app_tests.py b/seqr/views/react_app_tests.py index f363edcb09..09d177e82a 100644 --- a/seqr/views/react_app_tests.py +++ b/seqr/views/react_app_tests.py @@ -1,19 +1,81 @@ -from datetime import datetime from django.urls.base import reverse import mock +import responses from seqr.views.react_app import main_app, no_login_main_app from seqr.views.utils.terra_api_utils import TerraRefreshTokenFailedException from seqr.views.utils.test_utils import TEST_OAUTH2_PROVIDER, AuthenticationTestCase, AnvilAuthenticationTestCase, USER_FIELDS MOCK_GA_TOKEN = 'mock_ga_token' # nosec +FEATURE_UPDATE_FEED = """ + + + tag:github.com,2008:/broadinstitute/seqr/discussions/categories/feature-updates + + + Recent discussions in broadinstitute/seqr, category: feature-updates + 2023-11-08T15:11:40+00:00 + + tag:github.com,2008:5828412 + + Welcome to Feature Updates + 2023-11-08T15:11:31+00:00 + 2023-11-08T15:11:40+00:00 + + + hanars + https://github.com/hanars + + + <p dir="auto">Welcome to the seqr feature update discussion channel! This channel will be used for all announcements of new seqr functionality</p> + + + <h2 dir="auto">Welcome to our discussion forum!</h2> + <p dir="auto">We\xe2\x80\x99re using Discussions as a place to connect members of the seqr community with our development team and with one another. We hope that you:</p> + <ul dir="auto"> + <li>Ask questions</li> + <li>Describe issues that may be user errors instead of bugs</li> + <li>Answer and engage with one another</li> + <li>Discuss ideas for enhancements</li> + </ul> + + + + tag:github.com,2008:3913526 + + [ACTION REQUIRED] Upcoming Deprecation of older docker images and static JS/CSS assets + + 2022-03-03T16:25:56+00:00 + 2022-06-23T13:32:55+00:00 + + + sjahl + https://github.com/sjahl + + + <h3 dir="auto">What is happening?</h3> + <h4 dir="auto">Summary</h4> + <p dir="auto">Seqr is making updates to how our docker images work, and you will need to update your deployment by 2022/04/04. Please see the information below for more details.</p> + <h4 dir="auto">Slightly Longer Version</h4> + <p dir="auto">We\xe2\x80\x99re making two functional changes to the way seqr is packaged in Docker.</p> + <ol dir="auto"> + <li>We will be removing the statically built javascript/CSS/HTML assets that are checked into git. These will now be generated in the docker image when it\xe2\x80\x99s built, rather than provided in the source code tree.</li> + <li>New docker images will be built every time we release seqr. These images will no longer <code class="notranslate">git pull</code> to update themselves on startup.</li> + </ol> + <p dir="auto">However, if you are using a docker image from prior to when we made these changes, your installation still has the <code class="notranslate">git pull</code> update functionality. If you do nothing, your seqr installation may break after 2022/04/04 when it pulls down the version of the seqr code that has the assets removed. You will find instructions below on how to proceed.</p> + + + +""" + @mock.patch('seqr.views.react_app.DEBUG', False) +@mock.patch('seqr.utils.redis_utils.redis.StrictRedis') class AppPageTest(object): databases = ['default'] fixtures = ['users'] - def _check_page_html(self, response, user, user_key='user', vlm_enabled=False, user_email=None, user_fields=None, ga_token_id=None): + def _check_page_html(self, response, user, user_key='user', vlm_enabled=False, user_email=None, user_fields=None, ga_token_id=None, last_feature_update=None): user_fields = user_fields or USER_FIELDS self.assertEqual(response.status_code, 200) initial_json = self.get_initial_page_json(response) @@ -26,6 +88,7 @@ def _check_page_html(self, response, user, user_key='user', vlm_enabled=False, 'oauthLoginProvider': self.OAUTH_PROVIDER, 'vlmEnabled': vlm_enabled, 'warningMessages': [{'id': 1, 'header': 'Warning!', 'message': 'A sample warning'}], + 'lastFeatureUpdate': last_feature_update, }) self.assertEqual(self.get_initial_page_window('gaTrackingId', response), ga_token_id) @@ -39,14 +102,24 @@ def _check_page_html(self, response, user, user_key='user', vlm_enabled=False, @mock.patch('seqr.views.react_app.VLM_CLIENT_ID', 'abc123') @mock.patch('seqr.views.react_app.GA_TOKEN_ID', MOCK_GA_TOKEN) - def test_react_page(self): + @responses.activate + def test_react_page(self, mock_redis): url = reverse(main_app) self.check_require_login_no_policies(url, login_redirect_url='/login') + responses.add( + responses.GET, + 'https://github.com/broadinstitute/seqr/discussions/categories/feature-updates.atom', + body=FEATURE_UPDATE_FEED, + ) response = self.client.get(url) - self._check_page_html(response, 'test_user_no_policies', user_email='test_user_no_policy@test.com', ga_token_id=MOCK_GA_TOKEN, vlm_enabled=True) + self._check_page_html(response, 'test_user_no_policies', user_email='test_user_no_policy@test.com', ga_token_id=MOCK_GA_TOKEN, vlm_enabled=True, last_feature_update='2023-11-08T15:11:31+00:00') - def test_local_react_page(self): + mock_redis.return_value.set.assert_called_with('feature_updates_latest_date', '"2023-11-08T15:11:31+00:00"') + mock_redis.return_value.expire.assert_called_with('feature_updates_latest_date', 10800) + + @responses.activate + def test_local_react_page(self, mock_redis): url = reverse(no_login_main_app) response = self.client.get(url, HTTP_HOST='localhost:3000') self.assertEqual(response.status_code, 200) @@ -56,7 +129,8 @@ def test_local_react_page(self): self.assertContains(response, 'src="/app.js"') self.assertNotRegex(content, r']*>') - def test_no_login_react_page(self): + @responses.activate + def test_no_login_react_page(self, mock_redis): url = reverse(no_login_main_app) response = self.client.get(url) @@ -81,12 +155,15 @@ def test_no_login_react_page(self): response = self.client.get(url) self._check_page_html(response, 'test_user') - def test_react_page_additional_configs(self): + @responses.activate + def test_react_page_additional_configs(self, mock_redis): url = reverse(main_app) self.check_require_login_no_policies(url, login_redirect_url='/login') + mock_redis.return_value.get.return_value = '"2025-11-18T00:00:00Z"' + response = self.client.get(url) - self._check_page_html(response, 'test_user_no_policies') + self._check_page_html(response, 'test_user_no_policies', last_feature_update='2025-11-18T00:00:00Z') class LocalAppPageTest(AuthenticationTestCase, AppPageTest): diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py index 44a0d4c803..c39e47b609 100644 --- a/seqr/views/utils/orm_to_json_utils.py +++ b/seqr/views/utils/orm_to_json_utils.py @@ -10,7 +10,6 @@ from guardian.shortcuts import get_users_with_perms, get_groups_with_perms import json -from clickhouse_search.search import get_clickhouse_variant_annotations from panelapp.models import PaLocusList from reference_data.models import HumanPhenotypeOntology from seqr.models import GeneNote, VariantNote, VariantTag, VariantFunctionalData, SavedVariant, Family, CAN_VIEW, CAN_EDIT, \ @@ -420,6 +419,7 @@ def get_json_for_saved_variants(saved_variants, add_details=False, additional_mo def _add_clickhouse_annotations(results, genome_version): + from clickhouse_search.search import get_clickhouse_variant_annotations results_by_genome_version_dataset_type = defaultdict(lambda: defaultdict(list)) for result in results: dataset_type = result.pop('datasetType') diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js index 3fe2aace45..93918f9f33 100644 --- a/ui/redux/selectors.js +++ b/ui/redux/selectors.js @@ -41,6 +41,7 @@ export const getOauthLoginProvider = state => state.meta.oauthLoginProvider export const getVlmEnabled = state => state.meta.vlmEnabled export const getHijakEnabled = state => state.meta.hijakEnabled export const getWarningMessages = state => state.meta.warningMessages +export const getLastFeatureUpdate = state => state.meta.lastFeatureUpdate export const getSavedVariantsIsLoading = state => state.savedVariantsLoading.isLoading export const getSavedVariantsLoadingError = state => state.savedVariantsLoading.errorMessage export const getSearchesByHash = state => state.searchesByHash diff --git a/ui/shared/components/page/Header.jsx b/ui/shared/components/page/Header.jsx index f56839d732..93ee0326db 100644 --- a/ui/shared/components/page/Header.jsx +++ b/ui/shared/components/page/Header.jsx @@ -2,12 +2,12 @@ import React from 'react' import PropTypes from 'prop-types' import styled from 'styled-components' -import { Menu, Header, Dropdown } from 'semantic-ui-react' +import { Menu, Header, Dropdown, Label } from 'semantic-ui-react' import { connect } from 'react-redux' import { Link } from 'react-router-dom' import { updateUser } from 'redux/rootReducer' -import { getUser, getOauthLoginProvider } from 'redux/selectors' +import { getUser, getOauthLoginProvider, getLastFeatureUpdate } from 'redux/selectors' import { USER_NAME_FIELDS, LOCAL_LOGIN_URL, FEATURE_UPDATES_PATH } from 'shared/utils/constants' import UpdateButton from '../buttons/UpdateButton' @@ -18,7 +18,7 @@ const HeaderMenu = styled(Menu)` padding-right: 100px; ` -const PageHeader = React.memo(({ user, oauthLoginProvider, onSubmit }) => { +const PageHeader = React.memo(({ user, oauthLoginProvider, onSubmit, lastFeatureUpdate }) => { const loginUrl = oauthLoginProvider ? `/login/${oauthLoginProvider}` : LOCAL_LOGIN_URL return ( @@ -34,7 +34,11 @@ const PageHeader = React.memo(({ user, oauthLoginProvider, onSubmit }) => { , ] : null } - + + Feature Updates + {(lastFeatureUpdate && (new Date()).setMonth(new Date().getMonth() - 1) < new Date(lastFeatureUpdate)) && + } + {Object.keys(user).length ? [ ({ user: getUser(state), oauthLoginProvider: getOauthLoginProvider(state), + lastFeatureUpdate: getLastFeatureUpdate(state), }) const mapDispatchToProps = { diff --git a/vlm/match.py b/vlm/match.py index 3d2281a5c8..07cbe57e60 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -63,7 +63,7 @@ def _get_contact_url(chrom: str, pos: int, ref: str, alt: str, genome_build: str pos = lifted.position genome_build = liftover_genome_build genome_build = genome_build.replace('GRCh', '') - return f'{SEQR_BASE_URL}summary_data/variant_lookup?genomeVersion={genome_build}&variantId={chrom}-{pos}-{ref}-{alt}' + return f'{SEQR_BASE_URL}variant_lookup?genomeVersion={genome_build}&variantId={chrom}-{pos}-{ref}-{alt}' def _parse_match_query(query: dict) -> tuple[str, int, str, str, str]: diff --git a/vlm/test_vlm.py b/vlm/test_vlm.py index 0381198a41..7889cddbeb 100644 --- a/vlm/test_vlm.py +++ b/vlm/test_vlm.py @@ -52,7 +52,7 @@ async def test_match(self, mocked_responses): 'id': 'TestVLM', 'label': 'TestVLM browser', }, - 'url': 'https://test-seqr.org/summary_data/variant_lookup?genomeVersion=38&variantId=chr1-38724419-T-G', + 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=38&variantId=chr1-38724419-T-G', 'email': None, } ], @@ -124,7 +124,7 @@ async def test_match(self, mocked_responses): 'id': 'TestVLM', 'label': 'TestVLM browser', }, - 'url': 'https://test-seqr.org/summary_data/variant_lookup?genomeVersion=37&variantId=7-143270172-A-G', + 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=37&variantId=7-143270172-A-G', 'email': None, } ], @@ -195,7 +195,7 @@ async def test_match(self, mocked_responses): 'id': 'TestVLM', 'label': 'TestVLM browser', }, - 'url': 'https://test-seqr.org/summary_data/variant_lookup?genomeVersion=38&variantId=chr7-143270172-A-G', + 'url': 'https://test-seqr.org/variant_lookup?genomeVersion=38&variantId=chr7-143270172-A-G', 'email': None, } ],