Optimize transfer history filtering and indexes

app/api/routers/token.py

@@ -941,20 +941,16 @@ async def list_all_transfer_histories(
     """
 
     # Base query
-    from_address_tag = aliased(AccountTag)
-    to_address_tag = aliased(AccountTag)
-    stmt = (
-        select(IDXTransfer)
-        .outerjoin(
+    stmt = select(IDXTransfer)
+    if request_query.account_tag is not None:
+        from_address_tag = aliased(AccountTag)
+        to_address_tag = aliased(AccountTag)
+        stmt = stmt.outerjoin(
             from_address_tag,
             IDXTransfer.from_address == from_address_tag.account_address,
-        )
-        .outerjoin(
+        ).outerjoin(
             to_address_tag, IDXTransfer.to_address == to_address_tag.account_address
         )
-        .order_by(IDXTransfer.id)
-    )
-    if request_query.account_tag is not None:
         stmt = stmt.where(
             or_(
                 from_address_tag.account_tag == request_query.account_tag,
@@ -963,7 +959,7 @@ async def list_all_transfer_histories(
         )
 
     total = await async_session.scalar(
-        stmt.with_only_columns(func.count()).order_by(None)
+        stmt.with_only_columns(func.count(), maintain_column_froms=True).order_by(None)
     )
 
     # Filter
@@ -979,18 +975,12 @@ async def list_all_transfer_histories(
         )
     if request_query.transaction_hash is not None:
         stmt = stmt.where(
-            IDXTransfer.transaction_hash.like(
-                "%" + request_query.transaction_hash + "%"
-            )
+            IDXTransfer.transaction_hash == request_query.transaction_hash
         )
     if request_query.from_address is not None:
-        stmt = stmt.where(
-            IDXTransfer.from_address.like("%" + request_query.from_address + "%")
-        )
+        stmt = stmt.where(IDXTransfer.from_address == request_query.from_address)
     if request_query.to_address is not None:
-        stmt = stmt.where(
-            IDXTransfer.to_address.like("%" + request_query.to_address + "%")
-        )
+        stmt = stmt.where(IDXTransfer.to_address == request_query.to_address)
     if request_query.created_from is not None:
         stmt = stmt.where(IDXTransfer.created >= request_query.created_from)
     if request_query.created_to is not None:
@@ -1005,7 +995,7 @@ async def list_all_transfer_histories(
                 stmt = stmt.where(IDXTransfer.value <= request_query.value)
 
     count = await async_session.scalar(
-        stmt.with_only_columns(func.count()).order_by(None)
+        stmt.with_only_columns(func.count(), maintain_column_froms=True).order_by(None)
     )
 
     # Sort
@@ -1147,23 +1137,18 @@ async def list_token_transfer_histories(
         raise DataNotExistsError("token_address: %s" % token_address)
 
     # Base query
-    from_address_tag = aliased(AccountTag)
-    to_address_tag = aliased(AccountTag)
-    stmt = (
-        select(IDXTransfer)
-        .where(IDXTransfer.token_address == token_address)
-        .outerjoin(
+    stmt = select(IDXTransfer).where(IDXTransfer.token_address == token_address)
+
+    # Filter
+    if request_query.account_tag is not None:
+        from_address_tag = aliased(AccountTag)
+        to_address_tag = aliased(AccountTag)
+        stmt = stmt.outerjoin(
             from_address_tag,
             IDXTransfer.from_address == from_address_tag.account_address,
-        )
-        .outerjoin(
+        ).outerjoin(
             to_address_tag, IDXTransfer.to_address == to_address_tag.account_address
         )
-        .order_by(IDXTransfer.id)
-    )
-
-    # Filter
-    if request_query.account_tag is not None:
         stmt = stmt.where(
             or_(
                 from_address_tag.account_tag == request_query.account_tag,
@@ -1172,7 +1157,7 @@ async def list_token_transfer_histories(
         )
 
     total = await async_session.scalar(
-        stmt.with_only_columns(func.count()).order_by(None)
+        stmt.with_only_columns(func.count(), maintain_column_froms=True).order_by(None)
     )
 
     if request_query.source_event is not None:
@@ -1183,18 +1168,12 @@ async def list_token_transfer_histories(
         )
     if request_query.transaction_hash is not None:
         stmt = stmt.where(
-            IDXTransfer.transaction_hash.like(
-                "%" + request_query.transaction_hash + "%"
-            )
+            IDXTransfer.transaction_hash == request_query.transaction_hash
         )
     if request_query.from_address is not None:
-        stmt = stmt.where(
-            IDXTransfer.from_address.like("%" + request_query.from_address + "%")
-        )
+        stmt = stmt.where(IDXTransfer.from_address == request_query.from_address)
     if request_query.to_address is not None:
-        stmt = stmt.where(
-            IDXTransfer.to_address.like("%" + request_query.to_address + "%")
-        )
+        stmt = stmt.where(IDXTransfer.to_address == request_query.to_address)
     if request_query.created_from is not None:
         stmt = stmt.where(IDXTransfer.created >= request_query.created_from)
     if request_query.created_to is not None:
@@ -1209,7 +1188,7 @@ async def list_token_transfer_histories(
                 stmt = stmt.where(IDXTransfer.value <= request_query.value)
 
     count = await async_session.scalar(
-        stmt.with_only_columns(func.count()).order_by(None)
+        stmt.with_only_columns(func.count(), maintain_column_froms=True).order_by(None)
     )
 
     # Sort
@@ -1370,21 +1349,19 @@ async def search_transfer_histories(
             )
         )
     total = await async_session.scalar(
-        stmt.with_only_columns(func.count()).order_by(None)
+        stmt.with_only_columns(func.count(), maintain_column_froms=True).order_by(None)
     )
 
     if data.source_event is not None:
         stmt = stmt.where(IDXTransfer.source_event == data.source_event.value)
     if data.data is not None:
         stmt = stmt.where(cast(IDXTransfer.data, String).like("%" + data.data + "%"))
     if data.transaction_hash is not None:
-        stmt = stmt.where(
-            IDXTransfer.transaction_hash.like("%" + data.transaction_hash + "%")
-        )
+        stmt = stmt.where(IDXTransfer.transaction_hash == data.transaction_hash)
     if data.from_address is not None:
-        stmt = stmt.where(IDXTransfer.from_address.like("%" + data.from_address + "%"))
+        stmt = stmt.where(IDXTransfer.from_address == data.from_address)
     if data.to_address is not None:
-        stmt = stmt.where(IDXTransfer.to_address.like("%" + data.to_address + "%"))
+        stmt = stmt.where(IDXTransfer.to_address == data.to_address)
     if data.created_from is not None:
         stmt = stmt.where(
             IDXTransfer.created >= data.created_from.astimezone(timezone.utc)
@@ -1403,7 +1380,7 @@ async def search_transfer_histories(
                 stmt = stmt.where(IDXTransfer.value <= data.value)
 
     count = await async_session.scalar(
-        stmt.with_only_columns(func.count()).order_by(None)
+        stmt.with_only_columns(func.count(), maintain_column_froms=True).order_by(None)
     )
 
     sort_item = data.sort_item

app/model/db/idx_transfer.py

@@ -23,7 +23,7 @@
 from zoneinfo import ZoneInfo
 
 from pydantic import BaseModel
-from sqlalchemy import JSON, BigInteger, String
+from sqlalchemy import JSON, BigInteger, Index, String
 from sqlalchemy.orm import Mapped, mapped_column
 
 from app.config import TZ
@@ -55,6 +55,30 @@ class IDXTransfer(Base):
     """Token Transfer Events (INDEX)"""
 
     __tablename__ = "transfer"
+    __table_args__ = (
+        Index("ix_transfer_token_address_id", "token_address", "id"),
+        Index("ix_transfer_transaction_hash_id", "transaction_hash", "id"),
+        Index("ix_transfer_from_address_id", "from_address", "id"),
+        Index("ix_transfer_to_address_id", "to_address", "id"),
+        Index(
+            "ix_transfer_token_address_transaction_hash_id",
+            "token_address",
+            "transaction_hash",
+            "id",
+        ),
+        Index(
+            "ix_transfer_token_address_from_address_id",
+            "token_address",
+            "from_address",
+            "id",
+        ),
+        Index(
+            "ix_transfer_token_address_to_address_id",
+            "token_address",
+            "to_address",
+            "id",
+        ),
+    )
 
     # Sequence Id
     id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)

app/model/schema/token.py

@@ -178,9 +178,11 @@ class ListAllTransferHistoryQuery(BasePaginationQuery):
     )
     data: Optional[str] = Field(None, description="source event data")
     token_address: Optional[str] = Field(None, description="token address")
-    transaction_hash: Optional[str] = Field(None, description="transaction hash")
-    from_address: Optional[str] = Field(None, description="from address")
-    to_address: Optional[str] = Field(None, description="to address")
+    transaction_hash: Optional[str] = Field(
+        None, description="transaction hash (exact match)"
+    )
+    from_address: Optional[str] = Field(None, description="from address (exact match)")
+    to_address: Optional[str] = Field(None, description="to address (exact match)")
     value: Optional[int] = Field(None, description="value")
     value_operator: Optional[ValueOperator] = Field(
         ValueOperator.EQUAL,
@@ -202,9 +204,11 @@ class ListTokenTransferHistoryQuery(BasePaginationQuery):
         None, description="source event of transfer"
     )
     data: Optional[str] = Field(None, description="source event data")
-    transaction_hash: Optional[str] = Field(None, description="transaction hash")
-    from_address: Optional[str] = Field(None, description="from address")
-    to_address: Optional[str] = Field(None, description="to address")
+    transaction_hash: Optional[str] = Field(
+        None, description="transaction hash (exact match)"
+    )
+    from_address: Optional[str] = Field(None, description="from address (exact match)")
+    to_address: Optional[str] = Field(None, description="to address (exact match)")
     value: Optional[int] = Field(None, description="value")
     value_operator: Optional[ValueOperator] = Field(
         ValueOperator.EQUAL,
@@ -241,10 +245,14 @@ class SearchTransferHistoryRequest(BaseModel):
     )
     data: Optional[str] = Field(default=None, description="source event data")
     transaction_hash: Optional[str] = Field(
-        default=None, description="transaction hash"
+        default=None, description="transaction hash (exact match)"
+    )
+    from_address: Optional[str] = Field(
+        default=None, description="from address (exact match)"
+    )
+    to_address: Optional[str] = Field(
+        default=None, description="to address (exact match)"
     )
-    from_address: Optional[str] = Field(default=None, description="from address")
-    to_address: Optional[str] = Field(default=None, description="to address")
     created_from: Optional[datetime] = Field(
         default=None, description="created from datetime"
     )

docs/ibet_wallet_api.yaml

@@ -2914,29 +2914,29 @@ paths:
             anyOf:
               - type: string
               - type: 'null'
-            description: transaction hash
+            description: transaction hash (exact match)
             title: Transaction Hash
-          description: transaction hash
+          description: transaction hash (exact match)
         - name: from_address
           in: query
           required: false
           schema:
             anyOf:
               - type: string
               - type: 'null'
-            description: from address
+            description: from address (exact match)
             title: From Address
-          description: from address
+          description: from address (exact match)
         - name: to_address
           in: query
           required: false
           schema:
             anyOf:
               - type: string
               - type: 'null'
-            description: to address
+            description: to address (exact match)
             title: To Address
-          description: to address
+          description: to address (exact match)
         - name: value
           in: query
           required: false
@@ -3079,29 +3079,29 @@ paths:
             anyOf:
               - type: string
               - type: 'null'
-            description: transaction hash
+            description: transaction hash (exact match)
             title: Transaction Hash
-          description: transaction hash
+          description: transaction hash (exact match)
         - name: from_address
           in: query
           required: false
           schema:
             anyOf:
               - type: string
               - type: 'null'
-            description: from address
+            description: from address (exact match)
             title: From Address
-          description: from address
+          description: from address (exact match)
         - name: to_address
           in: query
           required: false
           schema:
             anyOf:
               - type: string
               - type: 'null'
-            description: to address
+            description: to address (exact match)
             title: To Address
-          description: to address
+          description: to address (exact match)
         - name: value
           in: query
           required: false
@@ -8889,19 +8889,19 @@ components:
             - type: string
             - type: 'null'
           title: Transaction Hash
-          description: transaction hash
+          description: transaction hash (exact match)
         from_address:
           anyOf:
             - type: string
             - type: 'null'
           title: From Address
-          description: from address
+          description: from address (exact match)
         to_address:
           anyOf:
             - type: string
             - type: 'null'
           title: To Address
-          description: to address
+          description: to address (exact match)
         created_from:
           anyOf:
             - type: string