diff --git a/.github/workflows/vlm-unit-tests.yaml b/.github/workflows/vlm-unit-tests.yaml index d83a7e0d80..35c1c631e8 100644 --- a/.github/workflows/vlm-unit-tests.yaml +++ b/.github/workflows/vlm-unit-tests.yaml @@ -1,4 +1,6 @@ name: VLM Unit Tests +permissions: + contents: read # Run the test suite on pushes (incl. merges) to master and dev # Run the test suite when a PR is opened, pushed to, or reopened @@ -19,7 +21,7 @@ on: - 'hail_search/fixtures/*' jobs: - vlm: + vlm_hail_backend: runs-on: ubuntu-latest container: hailgenetics/hail:0.2.128 @@ -36,5 +38,46 @@ jobs: export NODE_ID=TestVLM export MACHINE_MEM=24 export JAVA_OPTS_XSS=16M - coverage run --source="./vlm" --omit="./vlm/__main__.py" -m pytest vlm/ + coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py","./vlm/clickhouse_utils.py" -m pytest vlm/ coverage report --fail-under=95 + + vlm_clickhouse: + runs-on: ubuntu-latest + container: hailgenetics/hail:0.2.128 + + services: + clickhouse: + image: bitnami/clickhouse:latest + ports: + - 8123:8123 # HTTP interface + options: >- + --health-cmd "clickhouse-client --query 'SELECT 1'" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + --name clickhouse + env: + CLICKHOUSE_USER: clickhouse_test_user + CLICKHOUSE_PASSWORD: clickhouse_test_password + ALLOW_EMPTY_PASSWORD: no + + steps: + - uses: actions/checkout@v2 + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip wheel + pip install -r vlm/requirements-test.txt + - name: Set up Clickhouse Settings and Data + run: python3 vlm/setup_clickhouse_test_data.py clickhouse 8123 clickhouse_test_user clickhouse_test_password + - name: Run coverage tests + run: | + export SEQR_BASE_URL=https://test-seqr.org/ + export NODE_ID=TestVLM + export CLICKHOUSE_SERVICE_HOSTNAME=clickhouse + export CLICKHOUSE_SERVICE_PORT=8123 + export CLICKHOUSE_VLM_USERNAME=vlm_test_user + export CLICKHOUSE_VLM_PASSWORD=vlm_test_password + export CLICKHOUSE_DATABASE=test_seqr + coverage run --source="./vlm" --omit="./vlm/__main__.py","./vlm/setup_clickhouse_test_data.py","./vlm/hail_backend_utils.py" -m pytest vlm/ + coverage report --fail-under=95 + diff --git a/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py b/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py index a88e7cbf64..630bb5016b 100644 --- a/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py +++ b/clickhouse_search/migrations/0003_annotationsdiskmito_annotationsmito_entriesmito_and_more.py @@ -36,7 +36,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/MITO/annotations_disk', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/v0/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), @@ -62,7 +62,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/MITO/annotations_memory', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/v0/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), diff --git a/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py b/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py index 7a7dc55061..b9dc348dd3 100644 --- a/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py +++ b/clickhouse_search/migrations/0004_annotationsdisksv_annotationssv_entriessv.py @@ -42,7 +42,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/SV/annotations_disk', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/v0/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), @@ -74,7 +74,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/SV/annotations_memory', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/v0/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), diff --git a/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py b/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py index 31b25c4b02..cb3ca027d4 100644 --- a/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py +++ b/clickhouse_search/migrations/0005_annotationsdiskgcnv_annotationsgcnv_entriesgcnv_and_more.py @@ -36,7 +36,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/GCNV/annotations_disk', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/v0/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), @@ -63,7 +63,7 @@ class Migration(migrations.Migration): ], options={ 'db_table': 'GRCh38/GCNV/annotations_memory', - 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/v0/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), }, managers=[ ('objects', django.db.models.manager.Manager()), diff --git a/clickhouse_search/migrations/0013_annotationsdiskgcnv_annotationsdiskmito_and_more.py b/clickhouse_search/migrations/0013_annotationsdiskgcnv_annotationsdiskmito_and_more.py new file mode 100644 index 0000000000..2f0926d315 --- /dev/null +++ b/clickhouse_search/migrations/0013_annotationsdiskgcnv_annotationsdiskmito_and_more.py @@ -0,0 +1,196 @@ +import clickhouse_backend.models +import clickhouse_search.backend.engines +import clickhouse_search.backend.fields +from django.db import migrations +import django.db.models.manager + +from settings import CLICKHOUSE_IN_MEMORY_DIR, CLICKHOUSE_DATA_DIR + + +class Migration(migrations.Migration): + + dependencies = [ + ('clickhouse_search', '0012_delete_annotationsdiskgcnv_and_more'), + ] + + operations = [ + migrations.DeleteModel( + name='AnnotationsGcnv', + ), + migrations.DeleteModel( + name='AnnotationsMito', + ), + migrations.DeleteModel( + name='AnnotationsSv', + ), + migrations.CreateModel( + name='AnnotationsGcnv', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('pos', clickhouse_backend.models.UInt32Field()), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('end', clickhouse_backend.models.UInt32Field()), + ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], null=True)), ('position', clickhouse_backend.models.UInt32Field(blank=True, null=True))], db_column='rg37LocusEnd')), + ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), + ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), + ('num_exon', clickhouse_backend.models.UInt16Field(db_column='numExon')), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sv_callset', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field())]))])), + ], + options={ + 'db_table': 'GRCh38/GCNV/annotations_memory', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsMito', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('pos', clickhouse_backend.models.UInt32Field()), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('ref', clickhouse_backend.models.StringField()), + ('alt', clickhouse_backend.models.StringField()), + ('rsid', clickhouse_backend.models.StringField(blank=True, null=True)), + ('common_low_heteroplasmy', clickhouse_backend.models.BoolField(blank=True, db_column='commonLowHeteroplasmy', null=True)), + ('mitomap_pathogenic', clickhouse_backend.models.BoolField(blank=True, db_column='mitomapPathogenic', null=True)), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('apogee', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('haplogroup_defining', clickhouse_backend.models.BoolField(blank=True, null=True)), ('hmtvar', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mitotip', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'likely_pathogenic'), (1, 'possibly_pathogenic'), (2, 'possibly_benign'), (3, 'likely_benign')], null=True)), ('mut_taster', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'D'), (1, 'A'), (2, 'N'), (3, 'P')], null=True)), ('sift', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mlc', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_mito', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('gnomad_mito_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))])), ('helix', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('helix_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))]))])), + ('sorted_transcript_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('aminoAcids', clickhouse_backend.models.StringField(blank=True, null=True)), ('biotype', clickhouse_backend.models.StringField(blank=True, null=True)), ('canonical', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('codons', clickhouse_backend.models.StringField(blank=True, null=True)), ('consequenceTerms', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True))), ('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsc', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsp', clickhouse_backend.models.StringField(blank=True, null=True)), ('loftee', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('isLofNagnag', clickhouse_backend.models.BoolField(blank=True, null=True)), ('lofFilters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(blank=True, null=True)))])), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True)), ('transcriptId', clickhouse_backend.models.StringField()), ('transcriptRank', clickhouse_backend.models.UInt8Field())], db_column='sortedTranscriptConsequences')), + ], + options={ + 'db_table': 'GRCh38/MITO/annotations_memory', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsSv', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('pos', clickhouse_backend.models.UInt32Field()), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('end', clickhouse_backend.models.UInt32Field()), + ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], null=True)), ('position', clickhouse_backend.models.UInt32Field(blank=True, null=True))], db_column='rg37LocusEnd')), + ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), + ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), + ('algorithms', clickhouse_backend.models.StringField(low_cardinality=True)), + ('bothsides_support', clickhouse_backend.models.BoolField(db_column='bothsidesSupport')), + ('cpx_intervals', clickhouse_search.backend.fields.NestedField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('start', clickhouse_backend.models.UInt32Field()), ('end', clickhouse_backend.models.UInt32Field()), ('type', clickhouse_backend.models.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')]))], db_column='cpxIntervals')), + ('end_chrom', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='endChrom', null=True)), + ('sv_source_detail', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], null=True))], db_column='svSourceDetail')), + ('sv_type_detail', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'INS_iDEL'), (2, 'INVdel'), (3, 'INVdup'), (4, 'ME'), (5, 'ME:ALU'), (6, 'ME:LINE1'), (7, 'ME:SVA'), (8, 'dDUP'), (9, 'dDUP_iDEL'), (10, 'delINV'), (11, 'delINVdel'), (12, 'delINVdup'), (13, 'dupINV'), (14, 'dupINVdel'), (15, 'dupINVdup')], db_column='svTypeDetail', null=True)), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_svs', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field()), ('id', clickhouse_backend.models.StringField())]))])), + ], + options={ + 'db_table': 'GRCh38/SV/annotations_memory', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsDiskGcnv', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('pos', clickhouse_backend.models.UInt32Field()), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('end', clickhouse_backend.models.UInt32Field()), + ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], null=True)), ('position', clickhouse_backend.models.UInt32Field(blank=True, null=True))], db_column='rg37LocusEnd')), + ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), + ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), + ('num_exon', clickhouse_backend.models.UInt16Field(db_column='numExon')), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('sv_callset', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field())]))])), + ], + options={ + 'db_table': 'GRCh38/GCNV/annotations_disk', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/GCNV/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsDiskMito', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('pos', clickhouse_backend.models.UInt32Field()), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('ref', clickhouse_backend.models.StringField()), + ('alt', clickhouse_backend.models.StringField()), + ('rsid', clickhouse_backend.models.StringField(blank=True, null=True)), + ('common_low_heteroplasmy', clickhouse_backend.models.BoolField(blank=True, db_column='commonLowHeteroplasmy', null=True)), + ('mitomap_pathogenic', clickhouse_backend.models.BoolField(blank=True, db_column='mitomapPathogenic', null=True)), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('apogee', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('haplogroup_defining', clickhouse_backend.models.BoolField(blank=True, null=True)), ('hmtvar', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mitotip', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'likely_pathogenic'), (1, 'possibly_pathogenic'), (2, 'possibly_benign'), (3, 'likely_benign')], null=True)), ('mut_taster', clickhouse_backend.models.Enum8Field(blank=True, choices=[(0, 'D'), (1, 'A'), (2, 'N'), (3, 'P')], null=True)), ('sift', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True)), ('mlc', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_mito', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('gnomad_mito_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))])), ('helix', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field())])), ('helix_heteroplasmy', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('ac', clickhouse_backend.models.UInt32Field()), ('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('an', clickhouse_backend.models.UInt32Field()), ('max_hl', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9))]))])), + ('sorted_transcript_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('aminoAcids', clickhouse_backend.models.StringField(blank=True, null=True)), ('biotype', clickhouse_backend.models.StringField(blank=True, null=True)), ('canonical', clickhouse_backend.models.UInt8Field(blank=True, null=True)), ('codons', clickhouse_backend.models.StringField(blank=True, null=True)), ('consequenceTerms', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True))), ('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsc', clickhouse_backend.models.StringField(blank=True, null=True)), ('hgvsp', clickhouse_backend.models.StringField(blank=True, null=True)), ('loftee', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('isLofNagnag', clickhouse_backend.models.BoolField(blank=True, null=True)), ('lofFilters', clickhouse_backend.models.ArrayField(base_field=clickhouse_backend.models.StringField(blank=True, null=True)))])), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'transcript_ablation'), (2, 'splice_acceptor_variant'), (3, 'splice_donor_variant'), (4, 'stop_gained'), (5, 'frameshift_variant'), (6, 'stop_lost'), (7, 'start_lost'), (8, 'inframe_insertion'), (9, 'inframe_deletion'), (10, 'missense_variant'), (11, 'protein_altering_variant'), (12, 'splice_donor_5th_base_variant'), (13, 'splice_region_variant'), (14, 'splice_donor_region_variant'), (15, 'splice_polypyrimidine_tract_variant'), (16, 'incomplete_terminal_codon_variant'), (17, 'start_retained_variant'), (18, 'stop_retained_variant'), (19, 'synonymous_variant'), (20, 'coding_sequence_variant'), (21, 'mature_miRNA_variant'), (22, '5_prime_UTR_variant'), (23, '3_prime_UTR_variant'), (24, 'non_coding_transcript_exon_variant'), (25, 'intron_variant'), (26, 'NMD_transcript_variant'), (27, 'non_coding_transcript_variant'), (28, 'coding_transcript_variant'), (29, 'upstream_gene_variant'), (30, 'downstream_gene_variant'), (31, 'intergenic_variant'), (32, 'sequence_variant')], null=True)), ('transcriptId', clickhouse_backend.models.StringField()), ('transcriptRank', clickhouse_backend.models.UInt8Field())], db_column='sortedTranscriptConsequences')), + ], + options={ + 'db_table': 'GRCh38/MITO/annotations_disk', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/MITO/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='AnnotationsDiskSv', + fields=[ + ('key', clickhouse_search.backend.fields.UInt32FieldDeltaCodecField(primary_key=True, serialize=False)), + ('xpos', clickhouse_backend.models.UInt64Field()), + ('pos', clickhouse_backend.models.UInt32Field()), + ('variant_id', clickhouse_backend.models.StringField(db_column='variantId')), + ('lifted_over_pos', clickhouse_backend.models.UInt32Field(blank=True, db_column='liftedOverPos', null=True)), + ('chrom', clickhouse_search.backend.fields.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), + ('end', clickhouse_backend.models.UInt32Field()), + ('rg37_locus_end', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('contig', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], null=True)), ('position', clickhouse_backend.models.UInt32Field(blank=True, null=True))], db_column='rg37LocusEnd')), + ('lifted_over_chrom', clickhouse_search.backend.fields.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='liftedOverChrom', null=True)), + ('sv_type', clickhouse_search.backend.fields.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')], db_column='svType')), + ('predictions', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('strvctvre', clickhouse_backend.models.DecimalField(blank=True, decimal_places=5, max_digits=9, null=True))])), + ('sorted_gene_consequences', clickhouse_search.backend.fields.NestedField(base_fields=[('geneId', clickhouse_backend.models.StringField(blank=True, null=True)), ('majorConsequence', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'LOF'), (2, 'INTRAGENIC_EXON_DUP'), (3, 'PARTIAL_EXON_DUP'), (4, 'COPY_GAIN'), (5, 'DUP_PARTIAL'), (6, 'MSV_EXON_OVERLAP'), (7, 'INV_SPAN'), (8, 'UTR'), (9, 'PROMOTER'), (10, 'TSS_DUP'), (11, 'BREAKEND_EXONIC'), (12, 'INTRONIC'), (13, 'NEAREST_TSS')], null=True))], db_column='sortedGeneConsequences')), + ('algorithms', clickhouse_backend.models.StringField(low_cardinality=True)), + ('bothsides_support', clickhouse_backend.models.BoolField(db_column='bothsidesSupport')), + ('cpx_intervals', clickhouse_search.backend.fields.NestedField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')])), ('start', clickhouse_backend.models.UInt32Field()), ('end', clickhouse_backend.models.UInt32Field()), ('type', clickhouse_backend.models.Enum8Field(choices=[(1, 'gCNV_DEL'), (2, 'gCNV_DUP'), (3, 'BND'), (4, 'CPX'), (5, 'CTX'), (6, 'DEL'), (7, 'DUP'), (8, 'INS'), (9, 'INV'), (10, 'CNV')]))], db_column='cpxIntervals')), + ('end_chrom', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], db_column='endChrom', null=True)), + ('sv_source_detail', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('chrom', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21'), (22, '22'), (23, 'X'), (24, 'Y'), (25, 'M')], null=True))], db_column='svSourceDetail')), + ('sv_type_detail', clickhouse_backend.models.Enum8Field(blank=True, choices=[(1, 'INS_iDEL'), (2, 'INVdel'), (3, 'INVdup'), (4, 'ME'), (5, 'ME:ALU'), (6, 'ME:LINE1'), (7, 'ME:SVA'), (8, 'dDUP'), (9, 'dDUP_iDEL'), (10, 'delINV'), (11, 'delINVdel'), (12, 'delINVdup'), (13, 'dupINV'), (14, 'dupINVdel'), (15, 'dupINVdup')], db_column='svTypeDetail', null=True)), + ('populations', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('gnomad_svs', clickhouse_search.backend.fields.NamedTupleField(base_fields=[('af', clickhouse_backend.models.DecimalField(decimal_places=5, max_digits=9)), ('het', clickhouse_backend.models.UInt32Field()), ('hom', clickhouse_backend.models.UInt32Field()), ('id', clickhouse_backend.models.StringField())]))])), + ], + options={ + 'db_table': 'GRCh38/SV/annotations_disk', + 'engine': clickhouse_search.backend.engines.EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/SV/annotations', flatten_nested=0, primary_key='key'), + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('_overwrite_base_manager', django.db.models.manager.Manager()), + ], + ), + ] diff --git a/clickhouse_search/models.py b/clickhouse_search/models.py index 9432df87f7..c23171abcf 100644 --- a/clickhouse_search/models.py +++ b/clickhouse_search/models.py @@ -302,7 +302,7 @@ class Meta: db_table = 'GRCh38/SNV_INDEL/annotations_disk' engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/SNV_INDEL/annotations', primary_key='key', flatten_nested=0) -class AnnotationsMito(BaseAnnotationsMitoSnvIndel): +class BaseAnnotationsMito(BaseAnnotationsMitoSnvIndel): ANNOTATION_CONSTANTS = { 'chrom': 'M', 'liftedOverChrom': 'MT', @@ -358,11 +358,23 @@ class AnnotationsMito(BaseAnnotationsMitoSnvIndel): populations = NamedTupleField(POPULATION_FIELDS) sorted_transcript_consequences = NestedField(BaseAnnotationsMitoSnvIndel.TRANSCRIPTS_FIELDS, db_column='sortedTranscriptConsequences', group_by_key='geneId') + class Meta: + abstract = True + +class AnnotationsMito(BaseAnnotationsMito): + class Meta: db_table = 'GRCh38/MITO/annotations_memory' - engine = Join('ALL', 'INNER', 'key', join_use_nulls=1, flatten_nested=0) + engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/MITO/annotations', primary_key='key', flatten_nested=0) + +class AnnotationsDiskMito(BaseAnnotationsMito): + + class Meta: + db_table = 'GRCh38/MITO/annotations_disk' + engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/MITO/annotations', primary_key='key', flatten_nested=0) + -class AnnotationsSv(BaseAnnotationsSvGcnv): +class BaseAnnotationsSv(BaseAnnotationsSvGcnv): POPULATION_FIELDS = [ ('gnomad_svs', NamedTupleField([ ('af', models.DecimalField(max_digits=9, decimal_places=5)), @@ -390,11 +402,22 @@ class AnnotationsSv(BaseAnnotationsSvGcnv): sv_type_detail = models.Enum8Field(db_column='svTypeDetail', return_int=False, choices=SV_TYPE_DETAILS, null=True, blank=True) populations = NamedTupleField(POPULATION_FIELDS) + class Meta: + abstract = True + +class AnnotationsSv(BaseAnnotationsSv): + class Meta: db_table = 'GRCh38/SV/annotations_memory' - engine = Join('ALL', 'INNER', 'key', join_use_nulls=1, flatten_nested=0) + engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/SV/annotations', primary_key='key', flatten_nested=0) -class AnnotationsGcnv(BaseAnnotationsSvGcnv): +class AnnotationsDiskSv(BaseAnnotationsSv): + + class Meta: + db_table = 'GRCh38/SV/annotations_disk' + engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/SV/annotations', primary_key='key', flatten_nested=0) + +class BaseAnnotationsGcnv(BaseAnnotationsSvGcnv): POPULATION_FIELDS = [ ('sv_callset', NamedTupleField([ ('ac', models.UInt32Field()), @@ -416,9 +439,21 @@ class AnnotationsGcnv(BaseAnnotationsSvGcnv): num_exon = models.UInt16Field(db_column='numExon') populations = NamedTupleField(POPULATION_FIELDS) + class Meta: + abstract = True + +class AnnotationsGcnv(BaseAnnotationsGcnv): + class Meta: db_table = 'GRCh38/GCNV/annotations_memory' - engine = Join('ALL', 'LEFT', 'key', join_use_nulls=1, flatten_nested=0) + engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_IN_MEMORY_DIR}/GRCh38/GCNV/annotations', primary_key='key', flatten_nested=0) + +class AnnotationsDiskGcnv(BaseAnnotationsGcnv): + + class Meta: + db_table = 'GRCh38/GCNV/annotations_disk' + engine = EmbeddedRocksDB(0, f'{CLICKHOUSE_DATA_DIR}/GRCh38/GCNV/annotations', primary_key='key', flatten_nested=0) + class BaseClinvar(FixtureLoadableClickhouseModel): diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py index 31ed7fba11..851d3b1a66 100644 --- a/clickhouse_search/search.py +++ b/clickhouse_search/search.py @@ -2,6 +2,7 @@ from collections import defaultdict from django.contrib.postgres.aggregates import ArrayAgg from django.core.exceptions import ObjectDoesNotExist +from django.db import connections from django.db.models import F, Min, Q from django.db.models.functions import JSONObject import json @@ -11,7 +12,7 @@ ArrayDistinct, ArrayMap from clickhouse_search.models import ENTRY_CLASS_MAP, ANNOTATIONS_CLASS_MAP, TRANSCRIPTS_CLASS_MAP, KEY_LOOKUP_CLASS_MAP, \ BaseClinvar, BaseAnnotationsMitoSnvIndel, BaseAnnotationsGRCh37SnvIndel, BaseAnnotationsSvGcnv -from reference_data.models import GeneConstraint, Omim +from reference_data.models import GeneConstraint, Omim, GENOME_VERSION_LOOKUP from seqr.models import Sample, PhenotypePrioritization from seqr.utils.logging_utils import SeqrLogger from seqr.utils.search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \ @@ -531,3 +532,15 @@ def get_clickhouse_key_lookup(genome_version, dataset_type, variants_ids, revers batch = variants_ids[i:i + BATCH_SIZE] lookup.update(dict(key_lookup_class.objects.filter(variant_id__in=batch).values_list(*fields))) return lookup + + +def delete_clickhouse_project(project, dataset_type=None, **kwargs): + table_base = f'{GENOME_VERSION_LOOKUP[project.genome_version]}/{dataset_type}' + with connections['clickhouse_write'].cursor() as cursor: + cursor.execute(f'ALTER TABLE "{table_base}/entries" DROP PARTITION %s', [project.guid]) + cursor.execute(f'ALTER TABLE "{table_base}/project_gt_stats" DROP PARTITION %s', [project.guid]) + view_name = f'{table_base}/project_gt_stats_to_gt_stats_mv' + cursor.execute(f'SYSTEM REFRESH VIEW "{view_name}"') + cursor.execute(f'SYSTEM WAIT VIEW "{view_name}"') + cursor.execute(f'SYSTEM RELOAD DICTIONARY "{table_base}/gt_stats_dict"') + return f'Deleted all {dataset_type} search data for project {project.name}' diff --git a/seqr/urls.py b/seqr/urls.py index 07888d2b95..f93c3a603a 100644 --- a/seqr/urls.py +++ b/seqr/urls.py @@ -124,7 +124,8 @@ from seqr.views.apis.data_manager_api import elasticsearch_status, delete_index, \ update_rna_seq, load_rna_seq_sample_data, proxy_to_kibana, load_phenotype_prioritization_data, \ - validate_callset, get_loaded_projects, load_data, loading_vcfs, trigger_dag, proxy_to_luigi + validate_callset, get_loaded_projects, load_data, loading_vcfs, proxy_to_luigi, \ + trigger_delete_project, trigger_delete_family, trigger_update_search_reference_data from seqr.views.apis.report_api import \ anvil_export, \ family_metadata, \ @@ -339,7 +340,9 @@ 'data_management/loaded_projects/(?P[^/]+)/(?P[^/]+)/(?P[^/]+)': get_loaded_projects, 'data_management/load_data': load_data, 'data_management/add_igv': receive_bulk_igv_table_handler, - 'data_management/trigger_dag/(?P[^/]+)': trigger_dag, + 'data_management/trigger_delete_project': trigger_delete_project, + 'data_management/trigger_delete_family': trigger_delete_family, + 'data_management/trigger_update_search_reference_data': trigger_update_search_reference_data, 'summary_data/saved_variants/(?P[^/]+)': saved_variants_page, 'summary_data/hpo/(?P[^/]+)': hpo_summary_data, diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 65c8d599e7..5992cd9e2e 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -4,20 +4,19 @@ import gzip import json import os -import re import requests import urllib3 -from django.contrib.postgres.aggregates import ArrayAgg from django.core.exceptions import PermissionDenied from django.db.models import Max, F, Q from django.http.response import HttpResponse from django.views.decorators.csrf import csrf_exempt from requests.exceptions import ConnectionError as RequestConnectionError +from clickhouse_search.search import delete_clickhouse_project from seqr.utils.communication_utils import send_project_notification from seqr.utils.search.add_data_utils import prepare_data_loading_request, get_loading_samples_validator -from seqr.utils.search.utils import get_search_backend_status, delete_search_backend_data +from seqr.utils.search.utils import get_search_backend_status, delete_search_backend_data, backend_specific_call from seqr.utils.file_utils import file_iter, does_file_exist from seqr.utils.logging_utils import SeqrLogger from seqr.utils.middleware import ErrorsWarningsException @@ -27,7 +26,7 @@ from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS from seqr.views.utils.dataset_utils import load_rna_seq, load_phenotype_prioritization_data_file, RNA_DATA_TYPE_CONFIGS, \ post_process_rna_data, convert_django_meta_to_http_headers -from seqr.views.utils.file_utils import parse_file, get_temp_file_path, load_uploaded_file, persist_temp_file +from seqr.views.utils.file_utils import get_temp_file_path, load_uploaded_file, persist_temp_file from seqr.views.utils.json_utils import create_json_response, _to_snake_case from seqr.views.utils.json_to_orm_utils import update_model_from_json from seqr.views.utils.pedigree_info_utils import get_validated_related_individuals, JsonConstants @@ -425,24 +424,49 @@ def fetch_missing_vcf_samples(missing_vcf_samples): @data_manager_required -def trigger_dag(request, dag_id): +def trigger_delete_project(request): + request_json = json.loads(request.body) + project_guid = request_json.pop('project') + project = Project.objects.get(guid=project_guid) + return _trigger_data_update( + project, dag_id='DELETE_PROJECTS', clickhouse_func=delete_clickhouse_project, **request_json, + ) + + +@data_manager_required +def trigger_delete_family(request): + request_json = json.loads(request.body) + family_guid = request_json.pop('family') + project = Project.objects.get(family__guid=family_guid) + return _trigger_data_update(project, family_guid=family_guid, dag_id='DELETE_FAMILIES', **request_json) + + +@data_manager_required +def trigger_update_search_reference_data(request): + request_json = json.loads(request.body) + return _trigger_data_update(project=None, dag_id='UPDATE_REFERENCE_DATASETS', **request_json) + + +def _raise_backend_not_implemented(*args, **kwargs): + raise ErrorsWarningsException(['This functionality is not available in the current search backend']) + + +def _trigger_data_update(project, clickhouse_func=_raise_backend_not_implemented, **kwargs): + kwargs = {_to_snake_case(k): v for k, v in kwargs.items()} + info = backend_specific_call(_raise_backend_not_implemented, _trigger_dag, clickhouse_func)(project, **kwargs) + return create_json_response({'info': [info]}) + + +def _trigger_dag(project, dag_id=None, family_guid=None, **kwargs): if not is_airflow_enabled(): raise PermissionDenied() - request_json = json.loads(request.body) - project_guid = request_json.pop('project', None) - family_guid = request_json.pop('family', None) - kwargs = {_to_snake_case(k): v for k, v in request_json.items()} - project = None - if project_guid: - project = Project.objects.get(guid=project_guid) - elif family_guid: - project = Project.objects.get(family__guid=family_guid) + if family_guid: kwargs['family_guids'] = [family_guid] try: dag_variables = trigger_airflow_dag(dag_id, project, **kwargs) except Exception as e: - return create_json_response({'error': str(e)}, status=400) - return create_json_response({'info': [f'Triggered DAG {dag_id} with variables: {json.dumps(dag_variables)}']}) + raise ErrorsWarningsException([str(e)]) + return f'Triggered DAG {dag_id} with variables: {json.dumps(dag_variables)}' # Hop-by-hop HTTP response headers shouldn't be forwarded. diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 5021eadf80..8e14871f70 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -6,10 +6,11 @@ from requests import HTTPError import responses +from clickhouse_search.models import EntriesSnvIndel, ProjectGtStatsSnvIndel, AnnotationsSnvIndel from seqr.utils.communication_utils import _set_bulk_notification_stream from seqr.views.apis.data_manager_api import elasticsearch_status, delete_index, \ update_rna_seq, load_rna_seq_sample_data, load_phenotype_prioritization_data, validate_callset, loading_vcfs, \ - get_loaded_projects, trigger_dag, load_data + get_loaded_projects, load_data, trigger_delete_project, trigger_delete_family, trigger_update_search_reference_data from seqr.views.utils.orm_to_json_utils import _get_json_for_models from seqr.views.utils.test_utils import AuthenticationTestCase, AirflowTestCase, AirtableTest from seqr.utils.search.elasticsearch.es_utils_tests import urllib3_responses @@ -1515,10 +1516,12 @@ def _test_no_affected_family(self, url, body): Individual.objects.filter(guid='I000009_na20874').update(affected='A') @responses.activate - def test_trigger_dag(self): - self.check_data_manager_login(reverse(trigger_dag, args=['some_dag'])) + def test_trigger_delete_project(self): + url = reverse(trigger_delete_project) + self.check_data_manager_login(url) - self._test_trigger_single_dag( + self._test_trigger_search_data_update( + url, 'DELETE_PROJECTS', {'project': PROJECT_GUID, 'datasetType': 'SNV_INDEL'}, { @@ -1527,7 +1530,14 @@ def test_trigger_dag(self): 'reference_genome': 'GRCh37', } ) - self._test_trigger_single_dag( + + @responses.activate + def test_trigger_delete_family(self): + url = reverse(trigger_delete_family) + self.check_data_manager_login(url) + + self._test_trigger_search_data_update( + url, 'DELETE_FAMILIES', {'family': 'F000012_12', 'datasetType': 'MITO'}, { @@ -1538,29 +1548,30 @@ def test_trigger_dag(self): } ) + @responses.activate + def test_trigger_update_search_reference_data(self): + url = reverse(trigger_update_search_reference_data) + self.check_data_manager_login(url) + body = {'genomeVersion': '38', 'datasetType': 'SV'} - url = self._test_trigger_single_dag('UPDATE_REFERENCE_DATASETS',body,{ + self._test_trigger_search_data_update(url, 'UPDATE_REFERENCE_DATASETS', body, { 'projects_to_run': None, 'dataset_type': 'SV', 'reference_genome': 'GRCh38', }) - self._test_dag_trigger_errors(url, body) - - def _test_trigger_single_dag(self, dag_id, body, dag_variables): - responses.calls.reset() - self.set_up_one_dag(dag_id, variables=dag_variables) - - url = reverse(trigger_dag, args=[dag_id]) + def _test_trigger_search_data_update(self, url, dag_id, body, dag_variables): + self._setup_trigger_search_data_update(dag_id=dag_id, dag_variables=dag_variables) response = self.client.post(url, content_type='application/json', data=json.dumps(body)) - self._assert_expected_dag_trigger(response, dag_id, dag_variables) - return url + self._assert_expected_search_data_update(response, dag_id, dag_variables) - def _assert_expected_dag_trigger(self, response, dag_id, variables): - self.assertEqual(response.status_code, 403) + def _setup_trigger_search_data_update(self, *args, **kwargs): + return + + def _assert_expected_search_data_update(self, response, dag_id, variables): + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), {'errors': ['This functionality is not available in the current search backend'], 'warnings': None}) - def _test_dag_trigger_errors(self, url, body): - pass class LocalDataManagerAPITest(AuthenticationTestCase, DataManagerAPITest): fixtures = ['users', '1kg_project', 'reference_data'] @@ -1694,13 +1705,10 @@ def _assert_write_pedigree_error(self, response): def _test_validate_dataset_type(self, url): pass - def set_up_one_dag(self, *args, **kwargs): - pass - @mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') class AnvilDataManagerAPITest(AirflowTestCase, DataManagerAPITest): - fixtures = ['users', 'social_auth', '1kg_project', 'reference_data'] + fixtures = ['users', 'social_auth', '1kg_project', 'reference_data', 'clickhouse_search'] LOADING_PROJECT_GUID = NON_ANALYST_PROJECT_GUID CALLSET_DIR = 'gs://test_bucket' @@ -2009,6 +2017,51 @@ def _add_update_check_dag_responses(self, variables=None, **kwargs): return super()._add_update_check_dag_responses(**kwargs) + def _setup_trigger_search_data_update(self, *args, **kwargs): + Project.objects.filter(guid=PROJECT_GUID).update(genome_version='38') + + def _assert_expected_search_data_update(self, response, dag_id, variables): + if dag_id == 'DELETE_PROJECTS': + self._assert_expected_delete_project(response) + else: + super()._assert_expected_search_data_update(response, dag_id, variables) + + def _assert_expected_delete_project(self, response): + self.assertEqual(response.status_code, 200) + self.assertDictEqual(response.json(), { + 'info': ['Deleted all SNV_INDEL search data for project 1kg project n\xe5me with uni\xe7\xf8de'], + }) + self.assertEqual(EntriesSnvIndel.objects.filter(project_guid=PROJECT_GUID).count(), 0) + self.assertEqual(ProjectGtStatsSnvIndel.objects.filter(project_guid=PROJECT_GUID).count(), 0) + + updated_seqr_pops_by_key = dict(AnnotationsSnvIndel.objects.all().join_seqr_pop().values_list('key', 'seqrPop')) + self.assertDictEqual(updated_seqr_pops_by_key, { + 1: (2, 2, 1, 1), + 2: (1, 1, 0, 0), + 3: (0, 0, 0, 0), + 4: (0, 0, 0, 0), + 5: (1, 1, 0, 0), + 6: (0, 0, 0, 0), + 22: (0, 3, 0, 1), + }) + + def _assert_expected_airtable_errors(self, url): + responses.replace( + responses.GET, 'https://api.airtable.com/v0/app3Y97xtbbaOopVR/Samples', + json=INVALID_AIRTABLE_SAMPLE_RECORDS, status=200, + ) + response = self.client.get(url) + self.assertEqual(response.status_code, 400) + self.assertDictEqual(response.json(), { + 'error': 'The following samples are associated with misconfigured PDOs in Airtable: HG00731, NA21234', + }) + +@mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers') +class HailBackendDataManagerAPITest(AnvilDataManagerAPITest): + fixtures = ['users', 'social_auth', '1kg_project', 'reference_data'] + + CLICKHOUSE_HOSTNAME = '' + def _assert_update_check_airflow_calls(self, call_count, offset, update_check_path): if self.DAG_NAME != 'LOADING_PIPELINE': update_check_path = f'{self.MOCK_AIRFLOW_URL}/api/v1/variables/{self.DAG_NAME}' @@ -2020,26 +2073,27 @@ def set_up_one_dag(self, dag_id=None, **kwargs): self.DAG_NAME = dag_id super().set_up_one_dag(**kwargs) - def _assert_expected_dag_trigger(self, response, dag_id, variables): + def _assert_expected_search_data_update(self, response, dag_id, variables): self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), {'info': [f'Triggered DAG {dag_id} with variables: {json.dumps(variables)}']}) self.assert_airflow_calls(variables, 5) - def _test_dag_trigger_errors(self, url, body): + def _setup_trigger_search_data_update(self, *args, dag_id=None, dag_variables=None, **kwargs): + responses.calls.reset() + self.set_up_one_dag(dag_id, variables=dag_variables) + + def _test_trigger_search_data_update(self, url, dag_id, body, dag_variables): + super()._test_trigger_search_data_update(url, dag_id, body, dag_variables) + self.set_dag_trigger_error_response() response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 400) - self.assertDictEqual(response.json(), {'error': 'UPDATE_REFERENCE_DATASETS DAG is running and cannot be triggered again.'}) - - def _assert_expected_airtable_errors(self, url): - responses.replace( - responses.GET, 'https://api.airtable.com/v0/app3Y97xtbbaOopVR/Samples', - json=INVALID_AIRTABLE_SAMPLE_RECORDS, status=200, - ) - response = self.client.get(url) - self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), { - 'error': 'The following samples are associated with misconfigured PDOs in Airtable: HG00731, NA21234', + 'errors': [f'{dag_id} DAG is running and cannot be triggered again.'], 'warnings': None, }) + # DAG trigger not attempted when airflow is disabled + self.mock_airflow_url.__bool__.return_value = False + response = self.client.post(url, content_type='application/json', data=json.dumps(body)) + self.assertEqual(response.status_code, 403) diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index 7a2566e415..1100010398 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -625,8 +625,10 @@ def setUp(self): patcher = mock.patch('seqr.views.utils.airflow_utils.AuthorizedSession', mock.Mock(return_value=requests)) self.mock_authorized_session = patcher.start() self.addCleanup(patcher.stop) - patcher = mock.patch('seqr.views.utils.airflow_utils.AIRFLOW_WEBSERVER_URL', self.MOCK_AIRFLOW_URL) - patcher.start() + patcher = mock.patch('seqr.views.utils.airflow_utils.AIRFLOW_WEBSERVER_URL') + self.mock_airflow_url = patcher.start() + self.mock_airflow_url.__str__.return_value = self.MOCK_AIRFLOW_URL + self.mock_airflow_url.__bool__.return_value = True self.addCleanup(patcher.stop) patcher = mock.patch('seqr.views.utils.airflow_utils.safe_post_to_slack') self.mock_slack = patcher.start() diff --git a/ui/pages/DataManagement/DataManagement.jsx b/ui/pages/DataManagement/DataManagement.jsx index fd63d3bd2e..600e9044ac 100644 --- a/ui/pages/DataManagement/DataManagement.jsx +++ b/ui/pages/DataManagement/DataManagement.jsx @@ -12,7 +12,7 @@ import LoadData from './components/LoadData' import RnaSeq from './components/RnaSeq' import Users from './components/Users' import PhenotypePrioritization from './components/PhenotypePrioritization' -import TRIGGER_DAG_PAGES from './components/TriggerDagPages' +import TRIGGER_SEARCH_DATA_UPDATE_PAGES from './components/TriggerSearchDataUpdatePages' const IFRAME_STYLE = { position: 'fixed', left: '0', top: '95px' } @@ -51,7 +51,7 @@ const LOCAL_HAIL_SEARCH_DATA_MANAGEMENT_PAGES = [ const AIRFLOW_HAIL_SEARCH_DATA_MANAGEMENT_PAGES = [ ...DATA_MANAGEMENT_PAGES, - ...TRIGGER_DAG_PAGES, + ...TRIGGER_SEARCH_DATA_UPDATE_PAGES, ] const dataManagementPages = (user, elasticsearchEnabled) => { diff --git a/ui/pages/DataManagement/components/TriggerDagPages.jsx b/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx similarity index 65% rename from ui/pages/DataManagement/components/TriggerDagPages.jsx rename to ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx index db1b456fd2..7aa4ca250b 100644 --- a/ui/pages/DataManagement/components/TriggerDagPages.jsx +++ b/ui/pages/DataManagement/components/TriggerSearchDataUpdatePages.jsx @@ -11,6 +11,7 @@ import { DATASET_TYPE_MITO_CALLS, GENOME_VERSION_FIELD, } from 'shared/utils/constants' +import { snakecaseToTitlecase } from 'shared/utils/stringUtils' const DATASET_TYPE_FIELD = { name: 'datasetType', @@ -50,33 +51,33 @@ const REFERENCE_DATASET_FIELDS = [ DATASET_TYPE_FIELD, ] -const TriggerDagForm = ({ dagName, fields }) => ( +const TriggerSearchDataUpdateForm = ({ path, fields }) => ( ) -TriggerDagForm.propTypes = { - dagName: PropTypes.string, +TriggerSearchDataUpdateForm.propTypes = { + path: PropTypes.string, fields: PropTypes.arrayOf(PropTypes.object), } -const TriggerDeleteProjectsDag = () => ( - +const TriggerDeleteProjects = () => ( + ) -const TriggerDeleteFamiliesDag = () => ( - +const TriggerDeleteFamilies = () => ( + ) -const TriggerUpdateReferenceDatasetDag = () => ( - +const TriggerUpdateReferenceDataset = () => ( + ) export default [ - { path: 'delete_search_projects', component: TriggerDeleteProjectsDag }, - { path: 'delete_search_families', component: TriggerDeleteFamiliesDag }, - { path: 'update_search_reference_data', component: TriggerUpdateReferenceDatasetDag }, + { path: 'delete_search_project', component: TriggerDeleteProjects }, + { path: 'delete_search_family', component: TriggerDeleteFamilies }, + { path: 'update_search_reference_data', component: TriggerUpdateReferenceDataset }, ] diff --git a/vlm/clickhouse_utils.py b/vlm/clickhouse_utils.py new file mode 100644 index 0000000000..dd32573776 --- /dev/null +++ b/vlm/clickhouse_utils.py @@ -0,0 +1,25 @@ +import clickhouse_connect +import hail as hl +import os + +CLICKHOUSE_CONNECTION_PARAMS = { + 'host': os.environ.get('CLICKHOUSE_SERVICE_HOSTNAME'), + 'port': os.environ.get('CLICKHOUSE_SERVICE_PORT'), + 'username': os.environ.get('CLICKHOUSE_VLM_USERNAME'), + 'password': os.environ.get('CLICKHOUSE_VLM_PASSWORD'), + 'database': os.environ.get('CLICKHOUSE_DATABASE', 'seqr'), +} + + +def get_clickhouse_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, genome_build: str) -> hl.Struct: + locus = hl.eval(locus) + client = clickhouse_connect.get_client(**CLICKHOUSE_CONNECTION_PARAMS) + results = client.query( + "SELECT plus(gt_stats.1, gt_stats.2), plus(gt_stats.3, gt_stats.4) FROM (SELECT dictGet(%(dict_name)s, ('ac_wes', 'ac_wgs', 'hom_wes', 'hom_wgs'), key) AS gt_stats FROM %(table_name)s WHERE variantId=%(variant_id)s)", + parameters={ + 'variant_id': f'{locus.contig.replace("chr", "")}-{locus.position}-{ref}-{alt}', + 'table_name': f'{genome_build}/SNV_INDEL/key_lookup', + 'dict_name': f'{genome_build}/SNV_INDEL/gt_stats_dict', + }, + ).result_set + return results[0] if results else (0, 0) diff --git a/vlm/deploy/Dockerfile b/vlm/deploy/Dockerfile index 7788d5cd0b..bda44c60eb 100644 --- a/vlm/deploy/Dockerfile +++ b/vlm/deploy/Dockerfile @@ -2,6 +2,8 @@ FROM hailgenetics/hail:0.2.128 LABEL maintainer="Broad TGG" +RUN pip install --no-cache-dir clickhouse-connect==0.8.18 + WORKDIR /vlm # Application Code diff --git a/vlm/hail_backend_utils.py b/vlm/hail_backend_utils.py new file mode 100644 index 0000000000..6d12287870 --- /dev/null +++ b/vlm/hail_backend_utils.py @@ -0,0 +1,15 @@ +import hail as hl +import os + +VLM_DATA_DIR = os.environ.get('VLM_DATA_DIR') + + +def get_hail_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, genome_build: str) -> hl.Struct: + interval = hl.eval(hl.interval(locus, locus, includes_start=True, includes_end=True)) + ht = hl.read_table( + f'{VLM_DATA_DIR}/{genome_build}/SNV_INDEL/annotations.ht', _intervals=[interval], _filter_intervals=True, + ) + ht = ht.filter(ht.alleles == hl.array([ref, alt])) + + counts = ht.aggregate(hl.agg.take(ht.gt_stats, 1)) + return (counts[0].AC, counts[0].hom) if counts else (0, 0) diff --git a/vlm/match.py b/vlm/match.py index e5dc38e8b1..0795e0f87b 100644 --- a/vlm/match.py +++ b/vlm/match.py @@ -2,11 +2,15 @@ import hail as hl import os -VLM_DATA_DIR = os.environ.get('VLM_DATA_DIR') +from vlm.clickhouse_utils import get_clickhouse_variant_counts +from vlm.hail_backend_utils import get_hail_variant_counts + +CLICKHOUSE_ENABLED = bool(os.environ.get('CLICKHOUSE_SERVICE_HOSTNAME')) SEQR_BASE_URL = os.environ.get('SEQR_BASE_URL') VLM_DEFAULT_CONTACT_EMAIL = os.environ.get('VLM_DEFAULT_CONTACT_EMAIL') NODE_ID = os.environ.get('NODE_ID') + BEACON_HANDOVER_TYPE = { 'id': NODE_ID, 'label': f'{NODE_ID} browser' @@ -38,11 +42,12 @@ def get_variant_match(query: dict) -> dict: chrom, pos, ref, alt, genome_build = _parse_match_query(query) locus = hl.locus(chrom, pos, reference_genome=genome_build) - ac, hom = _get_variant_counts(locus, ref, alt, genome_build) + get_counts_func = get_clickhouse_variant_counts if CLICKHOUSE_ENABLED else get_hail_variant_counts + ac, hom = get_counts_func(locus, ref, alt, genome_build) liftover_genome_build = GENOME_VERSION_GRCh38 if genome_build == GENOME_VERSION_GRCh37 else GENOME_VERSION_GRCh37 liftover_locus = hl.liftover(locus, liftover_genome_build) - lift_ac, lift_hom = _get_variant_counts(liftover_locus, ref, alt, liftover_genome_build) + lift_ac, lift_hom = get_counts_func(liftover_locus, ref, alt, liftover_genome_build) url = _get_contact_url( chrom, pos, ref, alt, genome_build, liftover_genome_build, liftover_locus if lift_ac and not ac else None, @@ -88,17 +93,6 @@ def _parse_match_query(query: dict) -> tuple[str, int, str, str, str]: return chrom, start, query['referenceBases'], query['alternateBases'], genome_build -def _get_variant_counts(locus: hl.LocusExpression, ref: str, alt: str, genome_build: str) -> hl.Struct: - interval = hl.eval(hl.interval(locus, locus, includes_start=True, includes_end=True)) - ht = hl.read_table( - f'{VLM_DATA_DIR}/{genome_build}/SNV_INDEL/annotations.ht', _intervals=[interval], _filter_intervals=True, - ) - ht = ht.filter(ht.alleles == hl.array([ref, alt])) - - counts = ht.aggregate(hl.agg.take(ht.gt_stats, 1)) - return (counts[0].AC, counts[0].hom) if counts else (0, 0) - - def _format_results(ac: int, hom: int, url: str) -> dict: total = ac - hom # Homozygotes count twice toward the total AC result_sets = [ diff --git a/vlm/requirements-test.in b/vlm/requirements-test.in index f335034028..c63f835ebe 100644 --- a/vlm/requirements-test.in +++ b/vlm/requirements-test.in @@ -1,3 +1,4 @@ aioresponses coverage<5.2 pytest-aiohttp +clickhouse-connect==0.8.18 diff --git a/vlm/requirements-test.txt b/vlm/requirements-test.txt index df2cb3b32d..ccd84ac4f7 100644 --- a/vlm/requirements-test.txt +++ b/vlm/requirements-test.txt @@ -18,6 +18,8 @@ async-timeout==4.0.2 # via aiohttp attrs==23.1.0 # via aiohttp +clickhouse-connect==0.8.18 + # via -r vlm/requirements-test.in coverage==5.1 # via -r vlm/requirements-test.in exceptiongroup==1.1.3 diff --git a/vlm/setup_clickhouse_test_data.py b/vlm/setup_clickhouse_test_data.py new file mode 100644 index 0000000000..a50334040c --- /dev/null +++ b/vlm/setup_clickhouse_test_data.py @@ -0,0 +1,31 @@ +import clickhouse_connect +import sys + +def setup_clickhouse_test_data(host, port, username, password): + client = clickhouse_connect.get_client(host=host, port=port, username=username, password=password) + client.command('CREATE DATABASE test_seqr') + + client.command('CREATE TABLE test_seqr.`GRCh37/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId') + client.insert('GRCh37/SNV_INDEL/key_lookup', data=[['7-143270172-A-G', 1], ['1-39190091-T-G', 2]], database='test_seqr') + client.command( + 'CREATE DICTIONARY test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` (key UInt32, ac_wes UInt32, ac_wgs UInt32, hom_wes UInt32, hom_wgs UInt32) PRIMARY KEY key SOURCE(CLICKHOUSE(USER %s PASSWORD %s QUERY "SELECT * FROM VALUES ((1, 4104, 607, 1276, 232), (2, 7, 2, 2, 1))")) LIFETIME(0) LAYOUT(FLAT(MAX_ARRAY_SIZE 500000000))', + parameters=(username, password), + ) + + client.command('CREATE TABLE test_seqr.`GRCh38/SNV_INDEL/key_lookup` (`variantId` String, `key` UInt32 CODEC(Delta(8), ZSTD(1))) ENGINE = EmbeddedRocksDB(0) PRIMARY KEY variantId') + client.insert('GRCh38/SNV_INDEL/key_lookup', data=[['1-38724419-T-G', 1]], database='test_seqr') + client.command( + 'CREATE DICTIONARY test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` (key UInt32, ac_wes UInt32, ac_wgs UInt32, hom_wes UInt32, hom_wgs UInt32) PRIMARY KEY key SOURCE(CLICKHOUSE(USER %s PASSWORD %s QUERY "SELECT * FROM VALUES ((1, 18, 10, 3, 1))")) LIFETIME(0) LAYOUT(FLAT(MAX_ARRAY_SIZE 500000000))', + parameters=(username, password), + ) + + client.command("CREATE USER vlm_test_user IDENTIFIED WITH plaintext_password BY 'vlm_test_password'") + client.command('GRANT SELECT ON test_seqr.`GRCh37/SNV_INDEL/key_lookup` TO vlm_test_user') + client.command('GRANT SELECT ON test_seqr.`GRCh38/SNV_INDEL/key_lookup` TO vlm_test_user') + client.command('GRANT dictGet ON test_seqr.`GRCh37/SNV_INDEL/gt_stats_dict` TO vlm_test_user') + client.command('GRANT dictGet ON test_seqr.`GRCh38/SNV_INDEL/gt_stats_dict` TO vlm_test_user') + + +if __name__ == '__main__': + args = sys.argv[1:] + setup_clickhouse_test_data(*args)