From 2ebe180573864780fd750c164e586994a6388b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0tefan=20Baebler?= <319826+stefanb@users.noreply.github.com> Date: Sat, 20 Jun 2026 07:34:12 +0200 Subject: [PATCH 1/6] Add retry/backoff and request timeout to HTTP requests - Add create_session_with_retries() helper using urllib3 Retry strategy - Retry on 429, 500, 502, 503, 504 with exponential backoff - Overpass API uses higher backoff_factor (10s) due to rate limiting - All requests now have a 120s timeout to prevent indefinite hangs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- update.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/update.py b/update.py index 3ec0c6e..daf2616 100755 --- a/update.py +++ b/update.py @@ -2,11 +2,30 @@ import time import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry import os +REQUEST_TIMEOUT = 120 # seconds + +def create_session_with_retries(retries=3, backoff_factor=2, status_forcelist=(429, 500, 502, 503, 504)): + """Create a requests session with retry/backoff for transient failures.""" + session = requests.Session() + retry_strategy = Retry( + total=retries, + backoff_factor=backoff_factor, + status_forcelist=status_forcelist, + allowed_methods=["GET", "POST"], + ) + adapter = HTTPAdapter(max_retries=retry_strategy) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + def saveurl(url, filename, expectedContentType): print("Downloading ", url) - r = requests.get(url, allow_redirects=True) + session = create_session_with_retries() + r = session.get(url, allow_redirects=True, timeout=REQUEST_TIMEOUT) r.raise_for_status() actualContentType = r.headers['Content-Type'] @@ -58,14 +77,16 @@ def saveFromOverpassAPI( api_url: str = overpass_api_url, ): print(f"Requesting data from Overpass API. [url={api_url}]") + session = create_session_with_retries(retries=3, backoff_factor=10) request_headers = { **overpass_headers, "Accept": expectedContentType, } - response = requests.post( + response = session.post( url=api_url, data={"data": query}, headers=request_headers, + timeout=REQUEST_TIMEOUT, ) response.raise_for_status() print("Downloaded data from Overpass API.") From 13d372f3cb3724ffb14452cbbb5e94d1ea0766da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0tefan=20Baebler?= <319826+stefanb@users.noreply.github.com> Date: Sat, 20 Jun 2026 07:50:38 +0200 Subject: [PATCH 2/6] Configurable http methods, raise_on_status=false Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- update.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/update.py b/update.py index daf2616..ad7caec 100755 --- a/update.py +++ b/update.py @@ -8,14 +8,20 @@ REQUEST_TIMEOUT = 120 # seconds -def create_session_with_retries(retries=3, backoff_factor=2, status_forcelist=(429, 500, 502, 503, 504)): +def create_session_with_retries( + retries=3, + backoff_factor=2, + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=("GET", "HEAD", "OPTIONS"), +): """Create a requests session with retry/backoff for transient failures.""" session = requests.Session() retry_strategy = Retry( total=retries, backoff_factor=backoff_factor, status_forcelist=status_forcelist, - allowed_methods=["GET", "POST"], + allowed_methods=allowed_methods, + raise_on_status=False, ) adapter = HTTPAdapter(max_retries=retry_strategy) session.mount("https://", adapter) From b4a013eee2c98c7a7e59df34ee1fbc0d713b7dae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0tefan=20Baebler?= <319826+stefanb@users.noreply.github.com> Date: Sat, 20 Jun 2026 07:59:07 +0200 Subject: [PATCH 3/6] Explicitly enable retries on POST to Overpass API call Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update.py b/update.py index ad7caec..6fdce6d 100755 --- a/update.py +++ b/update.py @@ -83,7 +83,7 @@ def saveFromOverpassAPI( api_url: str = overpass_api_url, ): print(f"Requesting data from Overpass API. [url={api_url}]") - session = create_session_with_retries(retries=3, backoff_factor=10) + session = create_session_with_retries(retries=3, backoff_factor=10, allowed_methods=("POST",)) request_headers = { **overpass_headers, "Accept": expectedContentType, From 83bb6b53c067bf8367f9eaf5cfc767558d859a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0tefan=20Baebler?= <319826+stefanb@users.noreply.github.com> Date: Sat, 20 Jun 2026 08:00:41 +0200 Subject: [PATCH 4/6] Longer timeout for Overpass API calls Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- update.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/update.py b/update.py index 6fdce6d..1a08499 100755 --- a/update.py +++ b/update.py @@ -7,6 +7,7 @@ import os REQUEST_TIMEOUT = 120 # seconds +OVERPASS_REQUEST_TIMEOUT = 360 # seconds def create_session_with_retries( retries=3, @@ -92,7 +93,7 @@ def saveFromOverpassAPI( url=api_url, data={"data": query}, headers=request_headers, - timeout=REQUEST_TIMEOUT, + timeout=OVERPASS_REQUEST_TIMEOUT, ) response.raise_for_status() print("Downloaded data from Overpass API.") From c62162bede3dfa20d21293b3aad96426e0c1c1e6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 20 Jun 2026 06:14:35 +0000 Subject: [PATCH 5/6] Use context-managed requests sessions to avoid leaks --- __pycache__/update.cpython-312.pyc | Bin 0 -> 5368 bytes update.py | 30 ++++++++++++++--------------- 2 files changed, 15 insertions(+), 15 deletions(-) create mode 100644 __pycache__/update.cpython-312.pyc diff --git a/__pycache__/update.cpython-312.pyc b/__pycache__/update.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2efef8dcee353bcfcb5b0cb8db6a88ad3c37538a GIT binary patch literal 5368 zcmc&&TWlN06`dt_$>l@Ti;`%`c4_$~CL$@xdQkEsQ5?xJ9Lo>MjU&e;mYkuq63Jy| zS1*D_-6BC{AgC1-tyT2p0uAcIZjsM?^rL7$^Fu`n#B3D6MT-LdVl<;Lbepm7Ibgt;UdFV);Q2k(}o zC1oAAa_pNm$)|*I!Dw?y+qez$<8~~Jiw;!!!;RSX9yjj5_V>`Z6N}?6?7&X!`nhG? zjosr_xN6)(TnP?)NHwm8rv{!{Qk@WS-Fw1#4X%fNwb%=^j@09Z={AJu5ep(-c+iwN z>!<4b<@SB9)EI97$y)O(hv2Pl`)z?+-~vsLnCM7=(_L?jja?kXQbr>bUWLm*D?FQj0CERukUC; zdX1739$3v)+_2^NDf4Vh0jM#bv)wO`$D!xQ2s>p)QNWgc{WK+#M*NbWl6Pl_s;PdJ zsH&`_{d2N5314C5;g}SkRuTz+LZN<*N@-OlY0aOID5v)smu`^O%ZKU!Kz+OC0Qj=B^@ovvi(m<^HDamOw_D6 zsc9K?ARNxxSqmC4oM=v0w~vcNI;hBaY~Ls|dEH)KqwX=* z2Ln^sgoy>+E|m^lWxMY$l`Y3i1_L{kSQ_p||GE{C>_LaX#Kjj#vZ zu;#2&6+<)?B9sI7)|g(SfccgC-r#pPEmNZr03aZ2kNRDH$_O&1u2PNpJ&jd@sYqB(Px!|Ima_T6nPZufxTflMO->PIM%!|q zR?-^yMR05(Lv+hrOczF_S%SBGSJGrY1HuIR<7F8QWCj9GUC2;5tr@1HTPFy($K|0* zSB6H%qGM;z4_&x2rq`6*101OAAG&>#NSIJn=amde>%ttBHKNMM6ia0`Q1SxmWm0w(RsL#vJ>MPw27-8))v*WW&O>)f*S-q;^sSvtgGQ*m}3~ zxBb8FUlW5HV%6fr(){P5|L>lryss&WpoO8kj;h_?GH|ncWGz)@=QAv+&wM zd-kXqN0eDYGm@$XrHmZHBn(v;4go$wG$1s4QX)7A9xnw;j!(m#p$>oB%Vz%xaQc~8 zD6XW!Y7*=qnU<%+W}iT|L&{{5a$I635S|at&CLbj@KQn8Hj<7jnBe37_@qQtqMf`l zHXMuuELrCjmC)ee1egzPhIwb-cU}XV8kka*^mXU8vWX2G>*%!RY`wf9_0cll9)p>mlHIG=SF$pxL?)) zsgZ4p`TR5G`Pj6bs$u7SSPLs0`mSw?f%r28yBK2ahDX?K_Qh3(3=5~v6mt$Y>^WGY zfp`(XIW*k|JKeR7OYqHOBGwd3C!Q&=!p3Jl!K*p`4(zMmiG6^6Y;s1yH95OE$}Z^l zK^gsON3v~~OGzOEKNb-E396+07s~O|KX~zM$bSv&*wn1>(jHdHUS%9ZU8M}v(Rz}ziv#loL0$M(2Od~{YamOjkcxekGdl}gW zBv3{7vo_m3y~Ygn(F$Z{6v#n+7rs4o`QqT{X!IGhr3YAFHW^(|A&P5IK^B^!2if;S zj2vd<2#|nCgREU~7aC%(2AWov)6>ehkguv4B@I#jWe`%C{jwm5(v1XX-rfE2TOXhOXMdT6=<)z7@ z#oFTB@a!*Ip?>K0<{JZF3(#Dw2C((E+&S^#i9$_Nq0YB_c;(-|;YaLu-8aq0hhWYg_TP-P^z7i>&)jt@uuTqOST* z|7mc=H@xn9bH(@Os&52otWe#st(9jX+T%`!{e z{2>s?hAWKSu4B99zL+!pNp0AU5L1Q7v*zmE5S_POw_JH|cunkhY(s)?%@@jdT+NT;&v{~~-~C`Bpl|0jW#>k1 zZ#n}ETmcMY3+*G>pO|@AMuGS+Er%wQ*^nxSS+IbdD%>N*a6HoA8^nFFW5MoYQfDyM z6X^&_u`bdZPbB&yo$)ZZqj3=7{_Y;q)gA8*c1JoQ!EQqOf>NY69!w;9dZoUOV~KdL z6vpLCKsc#POL2LMz&6e*F*yx66_MkFolCaWJilk=1tQt941E>Wp8d`LSU_K-U+Rza zMuHMdEZCihlVE>WUr(?rg1e-?Sa)}4Phxukq`yPzJl544gg%L2cV};Zu)nvrCrEmw zm=x>pk0-i%wiiHVWef=++#gE$AWO8Sh`AQYFtpHoQk~Bpe(rJ%DOj{2Hu!Y5JDbKK zW<~R z=wdXQlH_zW3OK|}{1F3_fFS=KLaqFmk$*Argpq#(30QUOnV~VAe`9Fyb=`j9;@H^> zBcp*vooCrOW%pOT*8Ei>Wd4p&`az-&Ffu8}j)i74ITF$le=pOJn>7-$)r z@f32EQ!omt0@9myac6c%phJvng@9tXz2UsZrmMt#HTDU8O5c)R^z*&_ m8kXcGd`JFJ{?YVmUF*$rf8~8^eDfE4_50@m-}q+Z Date: Sat, 20 Jun 2026 06:15:01 +0000 Subject: [PATCH 6/6] Remove pycache artifact and ignore Python cache files --- .gitignore | 2 ++ __pycache__/update.cpython-312.pyc | Bin 5368 -> 0 bytes 2 files changed, 2 insertions(+) delete mode 100644 __pycache__/update.cpython-312.pyc diff --git a/.gitignore b/.gitignore index 6e2634e..c7d5012 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ venv .vscode +__pycache__/ +*.pyc diff --git a/__pycache__/update.cpython-312.pyc b/__pycache__/update.cpython-312.pyc deleted file mode 100644 index 2efef8dcee353bcfcb5b0cb8db6a88ad3c37538a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5368 zcmc&&TWlN06`dt_$>l@Ti;`%`c4_$~CL$@xdQkEsQ5?xJ9Lo>MjU&e;mYkuq63Jy| zS1*D_-6BC{AgC1-tyT2p0uAcIZjsM?^rL7$^Fu`n#B3D6MT-LdVl<;Lbepm7Ibgt;UdFV);Q2k(}o zC1oAAa_pNm$)|*I!Dw?y+qez$<8~~Jiw;!!!;RSX9yjj5_V>`Z6N}?6?7&X!`nhG? zjosr_xN6)(TnP?)NHwm8rv{!{Qk@WS-Fw1#4X%fNwb%=^j@09Z={AJu5ep(-c+iwN z>!<4b<@SB9)EI97$y)O(hv2Pl`)z?+-~vsLnCM7=(_L?jja?kXQbr>bUWLm*D?FQj0CERukUC; zdX1739$3v)+_2^NDf4Vh0jM#bv)wO`$D!xQ2s>p)QNWgc{WK+#M*NbWl6Pl_s;PdJ zsH&`_{d2N5314C5;g}SkRuTz+LZN<*N@-OlY0aOID5v)smu`^O%ZKU!Kz+OC0Qj=B^@ovvi(m<^HDamOw_D6 zsc9K?ARNxxSqmC4oM=v0w~vcNI;hBaY~Ls|dEH)KqwX=* z2Ln^sgoy>+E|m^lWxMY$l`Y3i1_L{kSQ_p||GE{C>_LaX#Kjj#vZ zu;#2&6+<)?B9sI7)|g(SfccgC-r#pPEmNZr03aZ2kNRDH$_O&1u2PNpJ&jd@sYqB(Px!|Ima_T6nPZufxTflMO->PIM%!|q zR?-^yMR05(Lv+hrOczF_S%SBGSJGrY1HuIR<7F8QWCj9GUC2;5tr@1HTPFy($K|0* zSB6H%qGM;z4_&x2rq`6*101OAAG&>#NSIJn=amde>%ttBHKNMM6ia0`Q1SxmWm0w(RsL#vJ>MPw27-8))v*WW&O>)f*S-q;^sSvtgGQ*m}3~ zxBb8FUlW5HV%6fr(){P5|L>lryss&WpoO8kj;h_?GH|ncWGz)@=QAv+&wM zd-kXqN0eDYGm@$XrHmZHBn(v;4go$wG$1s4QX)7A9xnw;j!(m#p$>oB%Vz%xaQc~8 zD6XW!Y7*=qnU<%+W}iT|L&{{5a$I635S|at&CLbj@KQn8Hj<7jnBe37_@qQtqMf`l zHXMuuELrCjmC)ee1egzPhIwb-cU}XV8kka*^mXU8vWX2G>*%!RY`wf9_0cll9)p>mlHIG=SF$pxL?)) zsgZ4p`TR5G`Pj6bs$u7SSPLs0`mSw?f%r28yBK2ahDX?K_Qh3(3=5~v6mt$Y>^WGY zfp`(XIW*k|JKeR7OYqHOBGwd3C!Q&=!p3Jl!K*p`4(zMmiG6^6Y;s1yH95OE$}Z^l zK^gsON3v~~OGzOEKNb-E396+07s~O|KX~zM$bSv&*wn1>(jHdHUS%9ZU8M}v(Rz}ziv#loL0$M(2Od~{YamOjkcxekGdl}gW zBv3{7vo_m3y~Ygn(F$Z{6v#n+7rs4o`QqT{X!IGhr3YAFHW^(|A&P5IK^B^!2if;S zj2vd<2#|nCgREU~7aC%(2AWov)6>ehkguv4B@I#jWe`%C{jwm5(v1XX-rfE2TOXhOXMdT6=<)z7@ z#oFTB@a!*Ip?>K0<{JZF3(#Dw2C((E+&S^#i9$_Nq0YB_c;(-|;YaLu-8aq0hhWYg_TP-P^z7i>&)jt@uuTqOST* z|7mc=H@xn9bH(@Os&52otWe#st(9jX+T%`!{e z{2>s?hAWKSu4B99zL+!pNp0AU5L1Q7v*zmE5S_POw_JH|cunkhY(s)?%@@jdT+NT;&v{~~-~C`Bpl|0jW#>k1 zZ#n}ETmcMY3+*G>pO|@AMuGS+Er%wQ*^nxSS+IbdD%>N*a6HoA8^nFFW5MoYQfDyM z6X^&_u`bdZPbB&yo$)ZZqj3=7{_Y;q)gA8*c1JoQ!EQqOf>NY69!w;9dZoUOV~KdL z6vpLCKsc#POL2LMz&6e*F*yx66_MkFolCaWJilk=1tQt941E>Wp8d`LSU_K-U+Rza zMuHMdEZCihlVE>WUr(?rg1e-?Sa)}4Phxukq`yPzJl544gg%L2cV};Zu)nvrCrEmw zm=x>pk0-i%wiiHVWef=++#gE$AWO8Sh`AQYFtpHoQk~Bpe(rJ%DOj{2Hu!Y5JDbKK zW<~R z=wdXQlH_zW3OK|}{1F3_fFS=KLaqFmk$*Argpq#(30QUOnV~VAe`9Fyb=`j9;@H^> zBcp*vooCrOW%pOT*8Ei>Wd4p&`az-&Ffu8}j)i74ITF$le=pOJn>7-$)r z@f32EQ!omt0@9myac6c%phJvng@9tXz2UsZrmMt#HTDU8O5c)R^z*&_ m8kXcGd`JFJ{?YVmUF*$rf8~8^eDfE4_50@m-}q+Z