eager fix (#2804)

* eager fix

* format

* Update app/dao/templates_dao.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update tests/app/dao/test_templates_dao.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* fix test

* Update tests/app/dao/test_templates_dao.py

Co-authored-by: Jimmy Royer <jimleroyer@gmail.com>

* Update app/dao/templates_dao.py

Co-authored-by: Jimmy Royer <jimleroyer@gmail.com>

* Update tests/app/dao/test_templates_dao.py

Co-authored-by: Jimmy Royer <jimleroyer@gmail.com>

---------

Co-authored-by: Ben Larabie <ben.larabie@blarabie-jtfk.home>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Jimmy Royer <jimleroyer@gmail.com>

Author: 4 people

app/dao/templates_dao.py

@@ -6,6 +6,7 @@
 from flask import current_app
 from notifications_utils.clients.redis import template_version_cache_key
 from sqlalchemy import asc, desc
+from sqlalchemy.orm import joinedload
 
 from app import db, redis_store
 from app.dao.dao_utils import VersionOptions, transactional, version_class
@@ -178,9 +179,17 @@ def dao_get_template_by_id(template_id, version=None, use_cache=False) -> Union[
 
 
 def dao_get_all_templates_for_service(service_id, template_type=None):
+    # Eager load fields required for serialization; otherwise lazy loading these will trigger thousands of individual queries.
+    query = Template.query.options(
+        joinedload("template_redacted"),
+        joinedload("template_category"),
+        joinedload("created_by"),
+        joinedload("service_letter_contact"),
+    )
+
     if template_type is not None:
         return (
-            Template.query.filter_by(
+            query.filter_by(
                 service_id=service_id,
                 template_type=template_type,
                 hidden=False,
@@ -194,7 +203,7 @@ def dao_get_all_templates_for_service(service_id, template_type=None):
         )
 
     return (
-        Template.query.filter_by(service_id=service_id, hidden=False, archived=False)
+        query.filter_by(service_id=service_id, hidden=False, archived=False)
         .order_by(
             asc(Template.name),
             asc(Template.template_type),

tests/app/dao/test_templates_dao.py

@@ -4,10 +4,11 @@
 
 import pytest
 from freezegun import freeze_time
+from sqlalchemy import event
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm.exc import NoResultFound
 
-from app import redis_store
+from app import db, redis_store
 from app.dao.templates_dao import (
     dao_create_template,
     dao_get_all_templates_for_service,
@@ -20,7 +21,7 @@
     dao_update_template_reply_to,
 )
 from app.models import Template, TemplateHistory, TemplateRedacted
-from app.schemas import template_schema
+from app.schemas import reduced_template_schema, template_schema
 from tests.app.db import create_letter_contact, create_template
 
 
@@ -232,6 +233,39 @@ def test_get_all_templates_for_service(service_factory):
     assert len(dao_get_all_templates_for_service(service_2.id)) == 2
 
 
+def test_get_all_templates_for_service_eager_loads_redaction_for_serialization(sample_service):
+    service_id = sample_service.id
+
+    for i in range(5):
+        create_template(
+            service=sample_service,
+            template_name=f"Sample Template {i}",
+            template_type="sms",
+            content=f"Template content {i}",
+        )
+
+    db.session.remove()
+
+    matching_statements = []
+
+    # Make sure that individual lazy queries are not called after serializing the templates.
+    def before_cursor_execute(_conn, _cursor, statement, _parameters, _context, _executemany):
+        normalized_statement = " ".join(statement.split())
+        if "FROM template_redacted" in normalized_statement and "WHERE template_redacted.template_id =" in normalized_statement:
+            matching_statements.append(normalized_statement)
+
+    writer_engine = db.get_engine(bind="writer")
+    event.listen(writer_engine, "before_cursor_execute", before_cursor_execute)
+    try:
+        templates = dao_get_all_templates_for_service(service_id)
+        # Trigger serialization and potential lazy loading if there are any remaining
+        reduced_template_schema.dump(templates, many=True)
+    finally:
+        event.remove(writer_engine, "before_cursor_execute", before_cursor_execute)
+
+    assert matching_statements == []
+
+
 def test_get_all_templates_for_service_is_alphabetised(sample_service):
     create_template(
         template_name="Sample Template 1",