diff --git a/backend/npdfhir/filters/organization_filter_set.py b/backend/npdfhir/filters/organization_filter_set.py index b66c4ca5..beb7e845 100644 --- a/backend/npdfhir/filters/organization_filter_set.py +++ b/backend/npdfhir/filters/organization_filter_set.py @@ -1,9 +1,9 @@ -from django_filters import rest_framework as filters from django.contrib.postgres.search import SearchVector from django.db.models import Q +from django_filters import rest_framework as filters -from ..models import Organization from ..mappings import addressUseMapping +from ..models import OrganizationByName from ..utils import parse_identifier_query @@ -38,7 +38,7 @@ class OrganizationFilterSet(filters.FilterSet): ) class Meta: - model = Organization + model = OrganizationByName fields = [ "name", "identifier", @@ -51,11 +51,7 @@ class Meta: ] def filter_name(self, queryset, name, value): - return ( - queryset.annotate(search=SearchVector("organizationtoname__name")) - .filter(search=value) - .distinct() - ) + return queryset.annotate(search=SearchVector("name")).filter(search=value).distinct() def filter_identifier(self, queryset, name, value): from uuid import UUID @@ -66,58 +62,62 @@ def filter_identifier(self, queryset, name, value): if system: # specific identifier search requested if system.upper() == "NPI": try: - queries = Q(clinicalorganization__npi__npi=int(identifier_id)) + queries = Q(organization__clinicalorganization__npi__npi=int(identifier_id)) except (ValueError, TypeError): pass # TODO: implement validationerror to show users that NPI must be an int else: # general identifier search requested try: - queries |= Q(clinicalorganization__npi__npi=int(identifier_id)) + queries |= Q(organization__clinicalorganization__npi__npi=int(identifier_id)) except (ValueError, TypeError): pass try: UUID(identifier_id) - queries |= Q(ein__ein_id=identifier_id) + queries |= Q(organization__ein__ein_id=identifier_id) except (ValueError, TypeError): pass - queries |= Q(clinicalorganization__organizationtootherid__other_id=identifier_id) + queries |= Q( + organization__clinicalorganization__organizationtootherid__other_id=identifier_id + ) return queryset.filter(queries).distinct() def filter_organization_type(self, queryset, name, value): return queryset.annotate( search=SearchVector( - "clinicalorganization__organizationtotaxonomy__nucc_code__display_name" + "organization__clinicalorganization__organizationtotaxonomy__nucc_code__display_name" ) ).filter(search=value) def filter_address(self, queryset, name, value): return queryset.annotate( search=SearchVector( - "organizationtoaddress__address__address_us__delivery_line_1", - "organizationtoaddress__address__address_us__delivery_line_2", - "organizationtoaddress__address__address_us__city_name", - "organizationtoaddress__address__address_us__state_code__abbreviation", - "organizationtoaddress__address__address_us__zipcode", + "organization__organizationtoaddress__address__address_us__delivery_line_1", + "organization__organizationtoaddress__address__address_us__delivery_line_2", + "organization__organizationtoaddress__address__address_us__city_name", + "organization__organizationtoaddress__address__address_us__state_code__abbreviation", + "organization__organizationtoaddress__address__address_us__zipcode", ) ).filter(search=value) def filter_address_city(self, queryset, name, value): return queryset.annotate( - search=SearchVector("organizationtoaddress__address__address_us__city_name") + search=SearchVector( + "organization__organizationtoaddress__address__address_us__city_name" + ) ).filter(search=value) def filter_address_state(self, queryset, name, value): return queryset.annotate( search=SearchVector( - "organizationtoaddress__address__address_us__state_code__abbreviation" + "organization__organizationtoaddress__address__address_us__state_code__abbreviation" ) ).filter(search=value) def filter_address_postalcode(self, queryset, name, value): return queryset.annotate( - search=SearchVector("organizationtoaddress__address__address_us__zipcode") + search=SearchVector("organization__organizationtoaddress__address__address_us__zipcode") ).filter(search=value) def filter_address_use(self, queryset, name, value): @@ -125,4 +125,4 @@ def filter_address_use(self, queryset, name, value): value = addressUseMapping.toNPD(value) else: value = -1 - return queryset.filter(organizationtoaddress__address_use_id=value) + return queryset.filter(organization__organizationtoaddress__address_use_id=value) diff --git a/backend/npdfhir/management/commands/explainquery.py b/backend/npdfhir/management/commands/explainquery.py new file mode 100644 index 00000000..911a8cdd --- /dev/null +++ b/backend/npdfhir/management/commands/explainquery.py @@ -0,0 +1,75 @@ +from django.core.management.base import BaseCommand + +from npdfhir.models import Organization, OrganizationByName + + +class Command(BaseCommand): + help = "Export a JSON schema, example JSON record, or both for a given FHIR entity" + + def original_query(self): + return ( + Organization.objects.all() + .prefetch_related( + "authorized_official", + "ein", + "organizationtoname_set", + "organizationtoaddress_set", + "organizationtoaddress_set__address", + "organizationtoaddress_set__address__address_us", + "organizationtoaddress_set__address__address_us__state_code", + "organizationtoaddress_set__address_use", + "authorized_official__individualtophone_set", + "authorized_official__individualtoname_set", + "authorized_official__individualtoemail_set", + "authorized_official__individualtoaddress_set", + "authorized_official__individualtoaddress_set__address__address_us", + "authorized_official__individualtoaddress_set__address__address_us__state_code", + "clinicalorganization", + "clinicalorganization__npi", + "clinicalorganization__organizationtootherid_set", + "clinicalorganization__organizationtootherid_set__other_id_type", + "clinicalorganization__organizationtotaxonomy_set", + "clinicalorganization__organizationtotaxonomy_set__nucc_code", + ) + .order_by("organizationtoname__name") + ) + + def materialized_view_query(self): + return ( + OrganizationByName.objects.all() + .prefetch_related( + "organization", + "organization__authorized_official", + "organization__ein", + "organization__organizationtoname_set", + "organization__organizationtoaddress_set", + "organization__organizationtoaddress_set__address", + "organization__organizationtoaddress_set__address__address_us", + "organization__organizationtoaddress_set__address__address_us__state_code", + "organization__organizationtoaddress_set__address_use", + "organization__authorized_official__individualtophone_set", + "organization__authorized_official__individualtoname_set", + "organization__authorized_official__individualtoemail_set", + "organization__authorized_official__individualtoaddress_set", + "organization__authorized_official__individualtoaddress_set__address__address_us", + "organization__authorized_official__individualtoaddress_set__address__address_us__state_code", + "organization__clinicalorganization", + "organization__clinicalorganization__npi", + "organization__clinicalorganization__organizationtootherid_set", + "organization__clinicalorganization__organizationtootherid_set__other_id_type", + "organization__clinicalorganization__organizationtotaxonomy_set", + "organization__clinicalorganization__organizationtotaxonomy_set__nucc_code", + ) + .order_by("name") + ) + + def handle(self, *args, **options): + for query in (self.original_query(), self.materialized_view_query()): + print("--- QUERY") + print(query[10:20].query) + print("--- EXPLAIN") + print(query[10:20].explain()) + print("---") + + # serializer = OrganizationSerializer(query[10:20], many=True) + # print(json.dumps(json.loads(JSONRenderer().render(serializer.data)), indent=2)) diff --git a/backend/npdfhir/management/commands/seedsystem.py b/backend/npdfhir/management/commands/seedsystem.py index b0f40dd0..2534749a 100644 --- a/backend/npdfhir/management/commands/seedsystem.py +++ b/backend/npdfhir/management/commands/seedsystem.py @@ -6,6 +6,7 @@ from django.db import IntegrityError from faker import Faker +from npdfhir.models import OrganizationByName from npdfhir.tests.fixtures import create_endpoint, create_organization, create_practitioner @@ -66,3 +67,5 @@ def handle(self, *args, **options): self.stdout.write(f"created Endpoint: {self.to_json(id=endpoint.id)}") self.generate_sample_organizations(25) + + OrganizationByName.refresh_materialized_view() diff --git a/backend/npdfhir/models.py b/backend/npdfhir/models.py index c4f60490..683dcf1c 100644 --- a/backend/npdfhir/models.py +++ b/backend/npdfhir/models.py @@ -5,7 +5,7 @@ # * Make sure each ForeignKey and OneToOneField has `on_delete` set to the desired behavior # * Remove `managed = False` lines if you wish to allow Django to create, modify, and delete the table # Feel free to rename the models, but don't rename db_table values or field names. -from django.db import models +from django.db import connection, models class Address(models.Model): @@ -589,6 +589,35 @@ class Meta: db_table = "organization_to_name" +class OrganizationByName(models.Model): + pk = models.CompositePrimaryKey("organization", "name") + + # `id` may not actually be unique in this table due to the underlying + # materialized view query which uses `LEFT OUTER JOIN` to collect a row for + # each name the organization has in organization_to_name. We need to + # include primary_key=True, though, to make Django happy. + organization = models.ForeignKey( + Organization, + models.DO_NOTHING, + blank=False, + null=False, + db_column="id", + unique=False, + ) + + # the sorting field from organization_to_name + name = models.CharField(max_length=1000) + + @classmethod + def refresh_materialized_view(cls): + with connection.cursor() as cursor: + cursor.execute(f"REFRESH MATERIALIZED VIEW {cls._meta.db_table};") + + class Meta: + managed = False + db_table = "organization_by_name" + + class OrganizationToOtherId(models.Model): pk = models.CompositePrimaryKey("npi", "other_id", "other_id_type_id", "issuer", "state_code") npi = models.ForeignKey(ClinicalOrganization, models.DO_NOTHING, db_column="npi") diff --git a/backend/npdfhir/serializers.py b/backend/npdfhir/serializers.py index 65757782..28e186fd 100644 --- a/backend/npdfhir/serializers.py +++ b/backend/npdfhir/serializers.py @@ -1,41 +1,42 @@ import sys +from datetime import datetime, timezone from django.urls import reverse from fhir.resources.R4B.address import Address from fhir.resources.R4B.bundle import Bundle +from fhir.resources.R4B.capabilitystatement import ( + CapabilityStatement, + CapabilityStatementImplementation, + CapabilityStatementRest, + CapabilityStatementRestResource, + CapabilityStatementRestResourceSearchParam, +) from fhir.resources.R4B.codeableconcept import CodeableConcept from fhir.resources.R4B.coding import Coding +from fhir.resources.R4B.contactdetail import ContactDetail from fhir.resources.R4B.contactpoint import ContactPoint from fhir.resources.R4B.endpoint import Endpoint from fhir.resources.R4B.humanname import HumanName from fhir.resources.R4B.identifier import Identifier from fhir.resources.R4B.location import Location as FHIRLocation -from fhir.resources.R4B.contactdetail import ContactDetail from fhir.resources.R4B.meta import Meta from fhir.resources.R4B.organization import Organization as FHIROrganization from fhir.resources.R4B.period import Period from fhir.resources.R4B.practitioner import Practitioner, PractitionerQualification from fhir.resources.R4B.practitionerrole import PractitionerRole from fhir.resources.R4B.reference import Reference -from fhir.resources.R4B.capabilitystatement import ( - CapabilityStatement, - CapabilityStatementRest, - CapabilityStatementRestResource, - CapabilityStatementRestResourceSearchParam, - CapabilityStatementImplementation, -) -from datetime import datetime, timezone from rest_framework import serializers -from .utils import get_schema_data, genReference from .models import ( IndividualToPhone, Location, Npi, Organization, + OrganizationByName, OrganizationToName, ProviderToOrganization, ) +from .utils import genReference, get_schema_data if "runserver" or "test" in sys.argv: from .cache import ( @@ -308,7 +309,10 @@ class Meta: model = Organization fields = "__all__" - def to_representation(self, instance): + def to_representation(self, instance: Organization | OrganizationByName): + if isinstance(instance, OrganizationByName): + instance = instance.organization + request = self.context.get("request") representation = super().to_representation(instance) organization = FHIROrganization() diff --git a/backend/npdfhir/tests/test_organization.py b/backend/npdfhir/tests/test_organization.py index 843d6c15..d6f74092 100644 --- a/backend/npdfhir/tests/test_organization.py +++ b/backend/npdfhir/tests/test_organization.py @@ -1,7 +1,9 @@ from django.urls import reverse from rest_framework import status -from ..models import Organization, OtherIdType + +from ..models import Organization, OrganizationByName, OtherIdType from .api_test_case import APITestCase +from .fixtures import create_legal_entity, create_organization from .helpers import ( assert_fhir_response, assert_has_results, @@ -9,8 +11,6 @@ extract_resource_names, ) -from .fixtures import create_organization, create_legal_entity - class OrganizationViewSetTestCase(APITestCase): @classmethod @@ -64,6 +64,9 @@ def setUpTestData(cls): cls.org_cumberland = create_organization(name="Cumberland") cls.orgs.append(cls.org_cumberland) + # refresh the sorting view used in FHIROrganizationViewSet + OrganizationByName.refresh_materialized_view() + return super().setUpTestData() def setUp(self): @@ -110,7 +113,7 @@ def test_list_in_default_order(self): def test_list_in_descending_order(self): url = reverse("fhir-organization-list") - response = self.client.get(url, {"_sort": "-organizationtoname__name"}) + response = self.client.get(url, {"_sort": "-name"}) assert_fhir_response(self, response) # Extract names @@ -118,7 +121,6 @@ def test_list_in_descending_order(self): names = extract_resource_names(response) sorted_names = [ - {}, "ZUNI HOME HEALTH CARE AGENCY", "ZEELAND COMMUNITY HOSPITAL", "YOUNGSTOWN ORTHOPAEDIC ASSOCIATES LTD", @@ -128,6 +130,7 @@ def test_list_in_descending_order(self): "YOAKUM COMMUNITY HOSPITAL", "YARMOUTH AUDIOLOGY", "TestNuccOrg", + "Joe Health Incorporated", ] self.assertEqual( diff --git a/backend/npdfhir/views.py b/backend/npdfhir/views.py index 7c7bcb78..ba588c1f 100644 --- a/backend/npdfhir/views.py +++ b/backend/npdfhir/views.py @@ -1,47 +1,44 @@ from uuid import UUID -from django.db.models import F, Value, CharField +from django.conf import settings +from django.db.models import CharField, F, Value from django.db.models.functions import Concat from django.http import HttpResponse from django.shortcuts import get_object_or_404 from django.utils.html import escape from django_filters.rest_framework import DjangoFilterBackend -from drf_spectacular.utils import extend_schema, OpenApiResponse +from drf_spectacular.utils import OpenApiResponse, extend_schema from rest_framework import viewsets -from rest_framework.views import APIView +from rest_framework.filters import OrderingFilter, SearchFilter from rest_framework.renderers import BrowsableAPIRenderer from rest_framework.response import Response -from rest_framework.filters import SearchFilter, OrderingFilter - -from .pagination import CustomPaginator -from .renderers import FHIRRenderer +from rest_framework.views import APIView from .filters.endpoint_filter_set import EndpointFilterSet from .filters.location_filter_set import LocationFilterSet from .filters.organization_filter_set import OrganizationFilterSet from .filters.practitioner_filter_set import PractitionerFilterSet from .filters.practitioner_role_filter_set import PractitionerRoleFilterSet - from .models import ( EndpointInstance, Location, Organization, + OrganizationByName, Provider, ProviderToLocation, ) - +from .pagination import CustomPaginator +from .renderers import FHIRRenderer from .serializers import ( BundleSerializer, + CapabilityStatementSerializer, EndpointSerializer, LocationSerializer, OrganizationSerializer, PractitionerRoleSerializer, PractitionerSerializer, - CapabilityStatementSerializer, ) -from django.conf import settings - DEBUG = settings.DEBUG @@ -334,7 +331,7 @@ class FHIROrganizationViewSet(viewsets.GenericViewSet): ViewSet for FHIR Organization resources """ - queryset = Organization.objects.none() + queryset = OrganizationByName.objects.none() if DEBUG: renderer_classes = [FHIRRenderer, BrowsableAPIRenderer] else: @@ -343,7 +340,7 @@ class FHIROrganizationViewSet(viewsets.GenericViewSet): filterset_class = OrganizationFilterSet pagination_class = CustomPaginator - ordering_fields = ["organizationtoname__name"] + ordering_fields = ["name"] # permission_classes = [permissions.IsAuthenticated] @extend_schema( @@ -361,30 +358,31 @@ def list(self, request): """ organizations = ( - Organization.objects.all() + OrganizationByName.objects.all() .prefetch_related( - "authorized_official", - "ein", - "organizationtoname_set", - "organizationtoaddress_set", - "organizationtoaddress_set__address", - "organizationtoaddress_set__address__address_us", - "organizationtoaddress_set__address__address_us__state_code", - "organizationtoaddress_set__address_use", - "authorized_official__individualtophone_set", - "authorized_official__individualtoname_set", - "authorized_official__individualtoemail_set", - "authorized_official__individualtoaddress_set", - "authorized_official__individualtoaddress_set__address__address_us", - "authorized_official__individualtoaddress_set__address__address_us__state_code", - "clinicalorganization", - "clinicalorganization__npi", - "clinicalorganization__organizationtootherid_set", - "clinicalorganization__organizationtootherid_set__other_id_type", - "clinicalorganization__organizationtotaxonomy_set", - "clinicalorganization__organizationtotaxonomy_set__nucc_code", + "organization", + "organization__authorized_official", + "organization__ein", + "organization__organizationtoname_set", + "organization__organizationtoaddress_set", + "organization__organizationtoaddress_set__address", + "organization__organizationtoaddress_set__address__address_us", + "organization__organizationtoaddress_set__address__address_us__state_code", + "organization__organizationtoaddress_set__address_use", + "organization__authorized_official__individualtophone_set", + "organization__authorized_official__individualtoname_set", + "organization__authorized_official__individualtoemail_set", + "organization__authorized_official__individualtoaddress_set", + "organization__authorized_official__individualtoaddress_set__address__address_us", + "organization__authorized_official__individualtoaddress_set__address__address_us__state_code", + "organization__clinicalorganization", + "organization__clinicalorganization__npi", + "organization__clinicalorganization__organizationtootherid_set", + "organization__clinicalorganization__organizationtootherid_set__other_id_type", + "organization__clinicalorganization__organizationtotaxonomy_set", + "organization__clinicalorganization__organizationtotaxonomy_set__nucc_code", ) - .order_by("organizationtoname__name") + .order_by("name") ) organizations = self.filter_queryset(organizations) diff --git a/flyway/sql/local_dev/R__sample_data.sql b/flyway/sql/local_dev/R__sample_data.sql index b78a6b31..2708785e 100644 --- a/flyway/sql/local_dev/R__sample_data.sql +++ b/flyway/sql/local_dev/R__sample_data.sql @@ -89666,3 +89666,4 @@ INSERT INTO npd.provider_to_other_id VALUES (1528061983, '222594672.0', 1, '33', -- PostgreSQL database dump complete -- +REFRESH MATERIALIZED VIEW "${apiSchema}"."organization_by_name"; diff --git a/flyway/sql/migrations/V15__fhir_organization_materialized_view.sql b/flyway/sql/migrations/V15__fhir_organization_materialized_view.sql new file mode 100644 index 00000000..ce7bb2c9 --- /dev/null +++ b/flyway/sql/migrations/V15__fhir_organization_materialized_view.sql @@ -0,0 +1,20 @@ +--- a materialized view covering the base query used by /fhir/Organization/ +CREATE MATERIALIZED VIEW IF NOT EXISTS + "${apiSchema}"."organization_by_name" AS +SELECT + "organization"."id", + "organization_to_name"."name" +FROM + "${apiSchema}".organization + LEFT OUTER JOIN "${apiSchema}".organization_to_name ON ( + "organization"."id" = "organization_to_name"."organization_id" + ) +ORDER BY + "organization_to_name"."name" ASC; + +-- an index on the column we always sort by +CREATE INDEX IF NOT EXISTS idx_organizationbyname_on_name ON "${apiSchema}"."organization_by_name" (name ASC); +-- a unique index to allow for use of REFRESH MATERIALIZED VIEW CONCURRENTLY +CREATE UNIQUE INDEX idx_organizationbyname_on_id_name ON "${apiSchema}"."organization_by_name" (id, name); + +REFRESH MATERIALIZED VIEW CONCURRENTLY "${apiSchema}"."organization_by_name"; \ No newline at end of file