diff --git a/batch/lib/http.py b/batch/lib/http.py new file mode 100644 index 000000000..40297205a --- /dev/null +++ b/batch/lib/http.py @@ -0,0 +1,33 @@ +""" +Copyright BOOSTRY Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +""" + +from requests.adapters import HTTPAdapter +from urllib3 import Retry + + +def get_retry_adapter() -> HTTPAdapter: + return HTTPAdapter( + max_retries=Retry( + total=3, + backoff_factor=1.0, + backoff_jitter=0.5, + status_forcelist=(500, 502, 503, 504), + allowed_methods=("GET",), + respect_retry_after_header=True, + raise_on_status=False, + ) + ) diff --git a/batch/sub_indexers/indexer_Company_List.py b/batch/sub_indexers/indexer_Company_List.py index f59105b3a..fda4d1aeb 100644 --- a/batch/sub_indexers/indexer_Company_List.py +++ b/batch/sub_indexers/indexer_Company_List.py @@ -22,16 +22,15 @@ import requests from pydantic import ValidationError -from requests.adapters import HTTPAdapter from sqlalchemy import delete from sqlalchemy.engine.create import create_engine from sqlalchemy.orm.session import Session -from urllib3 import Retry from app.config import COMPANY_LIST_URL, DATABASE_URL, REQUEST_TIMEOUT from app.model.db import Company from app.model.type import CompanyListItem from batch import log +from batch.lib.http import get_retry_adapter from batch.log import BatchLoggerAdapter process_name = "SUB:COMPANY-LIST" @@ -55,7 +54,7 @@ def process(self): return try: with requests.Session() as session: - adapter = HTTPAdapter(max_retries=Retry(3, allowed_methods=["GET"])) + adapter = get_retry_adapter() session.mount("http://", adapter) session.mount("https://", adapter) _resp = session.get( diff --git a/batch/sub_indexers/indexer_PublicInfo_PublicAccountList.py b/batch/sub_indexers/indexer_PublicInfo_PublicAccountList.py index 9e1dd19b6..5d720ab3c 100644 --- a/batch/sub_indexers/indexer_PublicInfo_PublicAccountList.py +++ b/batch/sub_indexers/indexer_PublicInfo_PublicAccountList.py @@ -23,15 +23,14 @@ import requests from eth_utils.address import to_checksum_address -from requests.adapters import HTTPAdapter from sqlalchemy import delete from sqlalchemy.engine.create import create_engine from sqlalchemy.orm.session import Session -from urllib3 import Retry from app.config import DATABASE_URL, PUBLIC_ACCOUNT_LIST_URL, REQUEST_TIMEOUT from app.model.db import PublicAccountList from batch import log +from batch.lib.http import get_retry_adapter from batch.log import BatchLoggerAdapter process_name = "SUB:PUBLIC-ACCOUNT-LIST" @@ -55,7 +54,7 @@ def process(self): return try: with requests.Session() as session: - adapter = HTTPAdapter(max_retries=Retry(3, allowed_methods=["GET"])) + adapter = get_retry_adapter() session.mount("http://", adapter) session.mount("https://", adapter) _resp = session.get( diff --git a/batch/sub_indexers/indexer_PublicInfo_TokenList.py b/batch/sub_indexers/indexer_PublicInfo_TokenList.py index ad6c66c50..7b365f71a 100644 --- a/batch/sub_indexers/indexer_PublicInfo_TokenList.py +++ b/batch/sub_indexers/indexer_PublicInfo_TokenList.py @@ -22,16 +22,15 @@ import requests from pydantic import ValidationError -from requests.adapters import HTTPAdapter from sqlalchemy import delete from sqlalchemy.engine.create import create_engine from sqlalchemy.orm.session import Session -from urllib3 import Retry from app.config import DATABASE_URL, REQUEST_TIMEOUT, TOKEN_LIST_URL from app.model.db import TokenList from app.model.type.token_list import TokenListItem from batch import log +from batch.lib.http import get_retry_adapter from batch.log import BatchLoggerAdapter process_name = "SUB:TOKEN-LIST" @@ -55,7 +54,7 @@ def process(self): LOG.warning("TOKEN_LIST_URL is not set") return with requests.Session() as session: - adapter = HTTPAdapter(max_retries=Retry(3, allowed_methods=["GET"])) + adapter = get_retry_adapter() session.mount("http://", adapter) session.mount("https://", adapter) _resp = session.get( diff --git a/tests/batch/sub_indexers/indexer_Company_List_test.py b/tests/batch/sub_indexers/indexer_Company_List_test.py index ae3c1a5a7..1d9a33012 100644 --- a/tests/batch/sub_indexers/indexer_Company_List_test.py +++ b/tests/batch/sub_indexers/indexer_Company_List_test.py @@ -1083,7 +1083,11 @@ async def test_error_1_1(self, processor: Processor, session: Session): # not succeed api @mock.patch("requests.Session.get") async def test_error_1_2( - self, mock_get: mock.MagicMock, processor: Processor, session: Session + self, + mock_get: mock.MagicMock, + processor: Processor, + session: Session, + caplog: pytest.LogCaptureFixture, ): # Prepare data _company = Company() @@ -1117,6 +1121,9 @@ async def test_error_1_2( select(Company).order_by(Company.created) ).all() assert len(_company_list) == 3 + assert 1 == caplog.record_tuples.count( + (LOG.name, logging.ERROR, "Failed to get company list") + ) # # not decode response