Add shared HTTP retry adapter for sub-indexers

batch/lib/http.py

@@ -0,0 +1,33 @@
+"""
+Copyright BOOSTRY Co., Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+SPDX-License-Identifier: Apache-2.0
+"""
+
+from requests.adapters import HTTPAdapter
+from urllib3 import Retry
+
+
+def get_retry_adapter() -> HTTPAdapter:
+    return HTTPAdapter(
+        max_retries=Retry(
+            total=3,
+            backoff_factor=1.0,
+            backoff_jitter=0.5,
+            status_forcelist=(500, 502, 503, 504),
+            allowed_methods=("GET",),
+            respect_retry_after_header=True,
+            raise_on_status=False,
+        )
+    )

batch/sub_indexers/indexer_Company_List.py

@@ -22,16 +22,15 @@
 
 import requests
 from pydantic import ValidationError
-from requests.adapters import HTTPAdapter
 from sqlalchemy import delete
 from sqlalchemy.engine.create import create_engine
 from sqlalchemy.orm.session import Session
-from urllib3 import Retry
 
 from app.config import COMPANY_LIST_URL, DATABASE_URL, REQUEST_TIMEOUT
 from app.model.db import Company
 from app.model.type import CompanyListItem
 from batch import log
+from batch.lib.http import get_retry_adapter
 from batch.log import BatchLoggerAdapter
 
 process_name = "SUB:COMPANY-LIST"
@@ -55,7 +54,7 @@ def process(self):
             return
         try:
             with requests.Session() as session:
-                adapter = HTTPAdapter(max_retries=Retry(3, allowed_methods=["GET"]))
+                adapter = get_retry_adapter()
                 session.mount("http://", adapter)
                 session.mount("https://", adapter)
                 _resp = session.get(

batch/sub_indexers/indexer_PublicInfo_PublicAccountList.py

@@ -23,15 +23,14 @@
 
 import requests
 from eth_utils.address import to_checksum_address
-from requests.adapters import HTTPAdapter
 from sqlalchemy import delete
 from sqlalchemy.engine.create import create_engine
 from sqlalchemy.orm.session import Session
-from urllib3 import Retry
 
 from app.config import DATABASE_URL, PUBLIC_ACCOUNT_LIST_URL, REQUEST_TIMEOUT
 from app.model.db import PublicAccountList
 from batch import log
+from batch.lib.http import get_retry_adapter
 from batch.log import BatchLoggerAdapter
 
 process_name = "SUB:PUBLIC-ACCOUNT-LIST"
@@ -55,7 +54,7 @@ def process(self):
             return
         try:
             with requests.Session() as session:
-                adapter = HTTPAdapter(max_retries=Retry(3, allowed_methods=["GET"]))
+                adapter = get_retry_adapter()
                 session.mount("http://", adapter)
                 session.mount("https://", adapter)
                 _resp = session.get(

batch/sub_indexers/indexer_PublicInfo_TokenList.py

@@ -22,16 +22,15 @@
 
 import requests
 from pydantic import ValidationError
-from requests.adapters import HTTPAdapter
 from sqlalchemy import delete
 from sqlalchemy.engine.create import create_engine
 from sqlalchemy.orm.session import Session
-from urllib3 import Retry
 
 from app.config import DATABASE_URL, REQUEST_TIMEOUT, TOKEN_LIST_URL
 from app.model.db import TokenList
 from app.model.type.token_list import TokenListItem
 from batch import log
+from batch.lib.http import get_retry_adapter
 from batch.log import BatchLoggerAdapter
 
 process_name = "SUB:TOKEN-LIST"
@@ -55,7 +54,7 @@ def process(self):
                 LOG.warning("TOKEN_LIST_URL is not set")
                 return
             with requests.Session() as session:
-                adapter = HTTPAdapter(max_retries=Retry(3, allowed_methods=["GET"]))
+                adapter = get_retry_adapter()
                 session.mount("http://", adapter)
                 session.mount("https://", adapter)
                 _resp = session.get(

tests/batch/sub_indexers/indexer_Company_List_test.py

@@ -1083,7 +1083,11 @@ async def test_error_1_1(self, processor: Processor, session: Session):
     # not succeed api
     @mock.patch("requests.Session.get")
     async def test_error_1_2(
-        self, mock_get: mock.MagicMock, processor: Processor, session: Session
+        self,
+        mock_get: mock.MagicMock,
+        processor: Processor,
+        session: Session,
+        caplog: pytest.LogCaptureFixture,
     ):
         # Prepare data
         _company = Company()
@@ -1117,6 +1121,9 @@ async def test_error_1_2(
             select(Company).order_by(Company.created)
         ).all()
         assert len(_company_list) == 3
+        assert 1 == caplog.record_tuples.count(
+            (LOG.name, logging.ERROR, "Failed to get company list")
+        )
 
     # <Error_2>
     # not decode response