[NDH-640][SPIKE] Replacing a base table with a materialized view

backend/npdfhir/filters/organization_filter_set.py

@@ -1,9 +1,9 @@
-from django_filters import rest_framework as filters
 from django.contrib.postgres.search import SearchVector
 from django.db.models import Q
+from django_filters import rest_framework as filters
 
-from ..models import Organization
 from ..mappings import addressUseMapping
+from ..models import OrganizationByName
 from ..utils import parse_identifier_query
 
 
@@ -38,7 +38,7 @@ class OrganizationFilterSet(filters.FilterSet):
     )
 
     class Meta:
-        model = Organization
+        model = OrganizationByName
         fields = [
             "name",
             "identifier",
@@ -51,11 +51,7 @@ class Meta:
         ]
 
     def filter_name(self, queryset, name, value):
-        return (
-            queryset.annotate(search=SearchVector("organizationtoname__name"))
-            .filter(search=value)
-            .distinct()
-        )
+        return queryset.annotate(search=SearchVector("name")).filter(search=value).distinct()
 
     def filter_identifier(self, queryset, name, value):
         from uuid import UUID
@@ -66,63 +62,67 @@ def filter_identifier(self, queryset, name, value):
         if system:  # specific identifier search requested
             if system.upper() == "NPI":
                 try:
-                    queries = Q(clinicalorganization__npi__npi=int(identifier_id))
+                    queries = Q(organization__clinicalorganization__npi__npi=int(identifier_id))
                 except (ValueError, TypeError):
                     pass  # TODO: implement validationerror to show users that NPI must be an int
         else:  # general identifier search requested
             try:
-                queries |= Q(clinicalorganization__npi__npi=int(identifier_id))
+                queries |= Q(organization__clinicalorganization__npi__npi=int(identifier_id))
             except (ValueError, TypeError):
                 pass
 
             try:
                 UUID(identifier_id)
-                queries |= Q(ein__ein_id=identifier_id)
+                queries |= Q(organization__ein__ein_id=identifier_id)
             except (ValueError, TypeError):
                 pass
 
-            queries |= Q(clinicalorganization__organizationtootherid__other_id=identifier_id)
+            queries |= Q(
+                organization__clinicalorganization__organizationtootherid__other_id=identifier_id
+            )
 
         return queryset.filter(queries).distinct()
 
     def filter_organization_type(self, queryset, name, value):
         return queryset.annotate(
             search=SearchVector(
-                "clinicalorganization__organizationtotaxonomy__nucc_code__display_name"
+                "organization__clinicalorganization__organizationtotaxonomy__nucc_code__display_name"
             )
         ).filter(search=value)
 
     def filter_address(self, queryset, name, value):
         return queryset.annotate(
             search=SearchVector(
-                "organizationtoaddress__address__address_us__delivery_line_1",
-                "organizationtoaddress__address__address_us__delivery_line_2",
-                "organizationtoaddress__address__address_us__city_name",
-                "organizationtoaddress__address__address_us__state_code__abbreviation",
-                "organizationtoaddress__address__address_us__zipcode",
+                "organization__organizationtoaddress__address__address_us__delivery_line_1",
+                "organization__organizationtoaddress__address__address_us__delivery_line_2",
+                "organization__organizationtoaddress__address__address_us__city_name",
+                "organization__organizationtoaddress__address__address_us__state_code__abbreviation",
+                "organization__organizationtoaddress__address__address_us__zipcode",
             )
         ).filter(search=value)
 
     def filter_address_city(self, queryset, name, value):
         return queryset.annotate(
-            search=SearchVector("organizationtoaddress__address__address_us__city_name")
+            search=SearchVector(
+                "organization__organizationtoaddress__address__address_us__city_name"
+            )
         ).filter(search=value)
 
     def filter_address_state(self, queryset, name, value):
         return queryset.annotate(
             search=SearchVector(
-                "organizationtoaddress__address__address_us__state_code__abbreviation"
+                "organization__organizationtoaddress__address__address_us__state_code__abbreviation"
             )
         ).filter(search=value)
 
     def filter_address_postalcode(self, queryset, name, value):
         return queryset.annotate(
-            search=SearchVector("organizationtoaddress__address__address_us__zipcode")
+            search=SearchVector("organization__organizationtoaddress__address__address_us__zipcode")
         ).filter(search=value)
 
     def filter_address_use(self, queryset, name, value):
         if value in addressUseMapping.keys():
             value = addressUseMapping.toNPD(value)
         else:
             value = -1
-        return queryset.filter(organizationtoaddress__address_use_id=value)
+        return queryset.filter(organization__organizationtoaddress__address_use_id=value)

backend/npdfhir/management/commands/explainquery.py

@@ -0,0 +1,75 @@
+from django.core.management.base import BaseCommand
+
+from npdfhir.models import Organization, OrganizationByName
+
+
+class Command(BaseCommand):
+    help = "Export a JSON schema, example JSON record, or both for a given FHIR entity"
+
+    def original_query(self):
+        return (
+            Organization.objects.all()
+            .prefetch_related(
+                "authorized_official",
+                "ein",
+                "organizationtoname_set",
+                "organizationtoaddress_set",
+                "organizationtoaddress_set__address",
+                "organizationtoaddress_set__address__address_us",
+                "organizationtoaddress_set__address__address_us__state_code",
+                "organizationtoaddress_set__address_use",
+                "authorized_official__individualtophone_set",
+                "authorized_official__individualtoname_set",
+                "authorized_official__individualtoemail_set",
+                "authorized_official__individualtoaddress_set",
+                "authorized_official__individualtoaddress_set__address__address_us",
+                "authorized_official__individualtoaddress_set__address__address_us__state_code",
+                "clinicalorganization",
+                "clinicalorganization__npi",
+                "clinicalorganization__organizationtootherid_set",
+                "clinicalorganization__organizationtootherid_set__other_id_type",
+                "clinicalorganization__organizationtotaxonomy_set",
+                "clinicalorganization__organizationtotaxonomy_set__nucc_code",
+            )
+            .order_by("organizationtoname__name")
+        )
+
+    def materialized_view_query(self):
+        return (
+            OrganizationByName.objects.all()
+            .prefetch_related(
+                "organization",
+                "organization__authorized_official",
+                "organization__ein",
+                "organization__organizationtoname_set",
+                "organization__organizationtoaddress_set",
+                "organization__organizationtoaddress_set__address",
+                "organization__organizationtoaddress_set__address__address_us",
+                "organization__organizationtoaddress_set__address__address_us__state_code",
+                "organization__organizationtoaddress_set__address_use",
+                "organization__authorized_official__individualtophone_set",
+                "organization__authorized_official__individualtoname_set",
+                "organization__authorized_official__individualtoemail_set",
+                "organization__authorized_official__individualtoaddress_set",
+                "organization__authorized_official__individualtoaddress_set__address__address_us",
+                "organization__authorized_official__individualtoaddress_set__address__address_us__state_code",
+                "organization__clinicalorganization",
+                "organization__clinicalorganization__npi",
+                "organization__clinicalorganization__organizationtootherid_set",
+                "organization__clinicalorganization__organizationtootherid_set__other_id_type",
+                "organization__clinicalorganization__organizationtotaxonomy_set",
+                "organization__clinicalorganization__organizationtotaxonomy_set__nucc_code",
+            )
+            .order_by("name")
+        )
+
+    def handle(self, *args, **options):
+        for query in (self.original_query(), self.materialized_view_query()):
+            print("--- QUERY")
+            print(query[10:20].query)
+            print("--- EXPLAIN")
+            print(query[10:20].explain())
+            print("---")
+
+        # serializer = OrganizationSerializer(query[10:20], many=True)
+        # print(json.dumps(json.loads(JSONRenderer().render(serializer.data)), indent=2))

backend/npdfhir/management/commands/seedsystem.py

@@ -6,6 +6,7 @@
 from django.db import IntegrityError
 from faker import Faker
 
+from npdfhir.models import OrganizationByName
 from npdfhir.tests.fixtures import create_endpoint, create_organization, create_practitioner
 
 
@@ -66,3 +67,5 @@ def handle(self, *args, **options):
             self.stdout.write(f"created Endpoint: {self.to_json(id=endpoint.id)}")
 
         self.generate_sample_organizations(25)
+
+        OrganizationByName.refresh_materialized_view()

backend/npdfhir/models.py

@@ -5,7 +5,7 @@
 #   * Make sure each ForeignKey and OneToOneField has `on_delete` set to the desired behavior
 #   * Remove `managed = False` lines if you wish to allow Django to create, modify, and delete the table
 # Feel free to rename the models, but don't rename db_table values or field names.
-from django.db import models
+from django.db import connection, models
 
 
 class Address(models.Model):
@@ -589,6 +589,35 @@ class Meta:
         db_table = "organization_to_name"
 
 
+class OrganizationByName(models.Model):
+    pk = models.CompositePrimaryKey("organization", "name")
+
+    # `id` may not actually be unique in this table due to the underlying
+    # materialized view query which uses `LEFT OUTER JOIN` to collect a row for
+    # each name the organization has in organization_to_name. We need to
+    # include primary_key=True, though, to make Django happy.
+    organization = models.ForeignKey(
+        Organization,
+        models.DO_NOTHING,
+        blank=False,
+        null=False,
+        db_column="id",
+        unique=False,
+    )
+
+    # the sorting field from organization_to_name
+    name = models.CharField(max_length=1000)
+
+    @classmethod
+    def refresh_materialized_view(cls):
+        with connection.cursor() as cursor:
+            cursor.execute(f"REFRESH MATERIALIZED VIEW {cls._meta.db_table};")
+
+    class Meta:
+        managed = False
+        db_table = "organization_by_name"
+
+
 class OrganizationToOtherId(models.Model):
     pk = models.CompositePrimaryKey("npi", "other_id", "other_id_type_id", "issuer", "state_code")
     npi = models.ForeignKey(ClinicalOrganization, models.DO_NOTHING, db_column="npi")

backend/npdfhir/serializers.py

@@ -1,41 +1,42 @@
 import sys
+from datetime import datetime, timezone
 
 from django.urls import reverse
 from fhir.resources.R4B.address import Address
 from fhir.resources.R4B.bundle import Bundle
+from fhir.resources.R4B.capabilitystatement import (
+    CapabilityStatement,
+    CapabilityStatementImplementation,
+    CapabilityStatementRest,
+    CapabilityStatementRestResource,
+    CapabilityStatementRestResourceSearchParam,
+)
 from fhir.resources.R4B.codeableconcept import CodeableConcept
 from fhir.resources.R4B.coding import Coding
+from fhir.resources.R4B.contactdetail import ContactDetail
 from fhir.resources.R4B.contactpoint import ContactPoint
 from fhir.resources.R4B.endpoint import Endpoint
 from fhir.resources.R4B.humanname import HumanName
 from fhir.resources.R4B.identifier import Identifier
 from fhir.resources.R4B.location import Location as FHIRLocation
-from fhir.resources.R4B.contactdetail import ContactDetail
 from fhir.resources.R4B.meta import Meta
 from fhir.resources.R4B.organization import Organization as FHIROrganization
 from fhir.resources.R4B.period import Period
 from fhir.resources.R4B.practitioner import Practitioner, PractitionerQualification
 from fhir.resources.R4B.practitionerrole import PractitionerRole
 from fhir.resources.R4B.reference import Reference
-from fhir.resources.R4B.capabilitystatement import (
-    CapabilityStatement,
-    CapabilityStatementRest,
-    CapabilityStatementRestResource,
-    CapabilityStatementRestResourceSearchParam,
-    CapabilityStatementImplementation,
-)
-from datetime import datetime, timezone
 from rest_framework import serializers
-from .utils import get_schema_data, genReference
 
 from .models import (
     IndividualToPhone,
     Location,
     Npi,
     Organization,
+    OrganizationByName,
     OrganizationToName,
     ProviderToOrganization,
 )
+from .utils import genReference, get_schema_data
 
 if "runserver" or "test" in sys.argv:
     from .cache import (
@@ -308,7 +309,10 @@ class Meta:
         model = Organization
         fields = "__all__"
 
-    def to_representation(self, instance):
+    def to_representation(self, instance: Organization | OrganizationByName):
+        if isinstance(instance, OrganizationByName):
+            instance = instance.organization
+
         request = self.context.get("request")
         representation = super().to_representation(instance)
         organization = FHIROrganization()

backend/npdfhir/tests/test_organization.py

@@ -1,16 +1,16 @@
 from django.urls import reverse
 from rest_framework import status
-from ..models import Organization, OtherIdType
+
+from ..models import Organization, OrganizationByName, OtherIdType
 from .api_test_case import APITestCase
+from .fixtures import create_legal_entity, create_organization
 from .helpers import (
     assert_fhir_response,
     assert_has_results,
     assert_pagination_limit,
     extract_resource_names,
 )
 
-from .fixtures import create_organization, create_legal_entity
-
 
 class OrganizationViewSetTestCase(APITestCase):
     @classmethod
@@ -64,6 +64,9 @@ def setUpTestData(cls):
         cls.org_cumberland = create_organization(name="Cumberland")
         cls.orgs.append(cls.org_cumberland)
 
+        # refresh the sorting view used in FHIROrganizationViewSet
+        OrganizationByName.refresh_materialized_view()
+
         return super().setUpTestData()
 
     def setUp(self):
@@ -110,15 +113,14 @@ def test_list_in_default_order(self):
 
     def test_list_in_descending_order(self):
         url = reverse("fhir-organization-list")
-        response = self.client.get(url, {"_sort": "-organizationtoname__name"})
+        response = self.client.get(url, {"_sort": "-name"})
         assert_fhir_response(self, response)
 
         # Extract names
         # Note: have to normalize the names to have python sorting match sql
         names = extract_resource_names(response)
 
         sorted_names = [
-            {},
             "ZUNI HOME HEALTH CARE AGENCY",
             "ZEELAND COMMUNITY HOSPITAL",
             "YOUNGSTOWN ORTHOPAEDIC ASSOCIATES LTD",
@@ -128,6 +130,7 @@ def test_list_in_descending_order(self):
             "YOAKUM COMMUNITY HOSPITAL",
             "YARMOUTH AUDIOLOGY",
             "TestNuccOrg",
+            "Joe Health Incorporated",
         ]
 
         self.assertEqual(