diff --git a/.gitignore b/.gitignore index 6e2634e..c7d5012 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ venv .vscode +__pycache__/ +*.pyc diff --git a/update.py b/update.py index 3ec0c6e..57b98d9 100755 --- a/update.py +++ b/update.py @@ -2,12 +2,38 @@ import time import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry import os +REQUEST_TIMEOUT = 120 # seconds +OVERPASS_REQUEST_TIMEOUT = 360 # seconds + +def create_session_with_retries( + retries=3, + backoff_factor=2, + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=("GET", "HEAD", "OPTIONS"), +): + """Create a requests session with retry/backoff for transient failures.""" + session = requests.Session() + retry_strategy = Retry( + total=retries, + backoff_factor=backoff_factor, + status_forcelist=status_forcelist, + allowed_methods=allowed_methods, + raise_on_status=False, + ) + adapter = HTTPAdapter(max_retries=retry_strategy) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + def saveurl(url, filename, expectedContentType): print("Downloading ", url) - r = requests.get(url, allow_redirects=True) - r.raise_for_status() + with create_session_with_retries() as session: + r = session.get(url, allow_redirects=True, timeout=REQUEST_TIMEOUT) + r.raise_for_status() actualContentType = r.headers['Content-Type'] @@ -58,16 +84,18 @@ def saveFromOverpassAPI( api_url: str = overpass_api_url, ): print(f"Requesting data from Overpass API. [url={api_url}]") - request_headers = { - **overpass_headers, - "Accept": expectedContentType, - } - response = requests.post( - url=api_url, - data={"data": query}, - headers=request_headers, - ) - response.raise_for_status() + with create_session_with_retries(retries=3, backoff_factor=10, allowed_methods=("POST",)) as session: + request_headers = { + **overpass_headers, + "Accept": expectedContentType, + } + response = session.post( + url=api_url, + data={"data": query}, + headers=request_headers, + timeout=OVERPASS_REQUEST_TIMEOUT, + ) + response.raise_for_status() print("Downloaded data from Overpass API.") actualContentType = response.headers['Content-Type'].split(';')[0].strip()