Add retry/backoff and request timeout to HTTP requests

.gitignore

@@ -1,3 +1,5 @@
 
 venv
 .vscode
+__pycache__/
+*.pyc

update.py

@@ -2,12 +2,38 @@
 
 import time
 import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 import os
 
+REQUEST_TIMEOUT = 120  # seconds
+OVERPASS_REQUEST_TIMEOUT = 360  # seconds
+
+def create_session_with_retries(
+    retries=3,
+    backoff_factor=2,
+    status_forcelist=(429, 500, 502, 503, 504),
+    allowed_methods=("GET", "HEAD", "OPTIONS"),
+):
+    """Create a requests session with retry/backoff for transient failures."""
+    session = requests.Session()
+    retry_strategy = Retry(
+        total=retries,
+        backoff_factor=backoff_factor,
+        status_forcelist=status_forcelist,
+        allowed_methods=allowed_methods,
+        raise_on_status=False,
+    )
+    adapter = HTTPAdapter(max_retries=retry_strategy)
+    session.mount("https://", adapter)
+    session.mount("http://", adapter)
+    return session
+
 def saveurl(url, filename, expectedContentType):
     print("Downloading ", url)
-    r = requests.get(url, allow_redirects=True)
-    r.raise_for_status()
+    with create_session_with_retries() as session:
+        r = session.get(url, allow_redirects=True, timeout=REQUEST_TIMEOUT)
+        r.raise_for_status()
 
     actualContentType = r.headers['Content-Type']
 
@@ -58,16 +84,18 @@ def saveFromOverpassAPI(
     api_url: str = overpass_api_url,
 ):
     print(f"Requesting data from Overpass API. [url={api_url}]")
-    request_headers = {
-        **overpass_headers,
-        "Accept": expectedContentType,
-    }
-    response = requests.post(
-        url=api_url,
-        data={"data": query},
-        headers=request_headers,
-    )
-    response.raise_for_status()
+    with create_session_with_retries(retries=3, backoff_factor=10, allowed_methods=("POST",)) as session:
+        request_headers = {
+            **overpass_headers,
+            "Accept": expectedContentType,
+        }
+        response = session.post(
+            url=api_url,
+            data={"data": query},
+            headers=request_headers,
+            timeout=OVERPASS_REQUEST_TIMEOUT,
+        )
+        response.raise_for_status()
     print("Downloaded data from Overpass API.")
 
     actualContentType = response.headers['Content-Type'].split(';')[0].strip()