From 83db89bd460810f65eae861d1db910605a76011b Mon Sep 17 00:00:00 2001 From: rpwagner Date: Fri, 28 Mar 2025 17:03:16 -0700 Subject: [PATCH 1/6] initial replacement of urllib with urllib3 --- environment.yml | 1 + lib/cartopy/io/__init__.py | 5 +++-- lib/cartopy/io/srtm.py | 4 ++-- pyproject.toml | 1 + 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/environment.yml b/environment.yml index 8c5d5d3b3..8f1e46b9f 100644 --- a/environment.yml +++ b/environment.yml @@ -14,6 +14,7 @@ dependencies: - pyshp>=2.3 - pyproj>=3.3.1 - packaging>=21 + - urllib3>=2.3 # The testing label has the proper version of freetype included - conda-forge/label/testing::matplotlib-base>=3.6 diff --git a/lib/cartopy/io/__init__.py b/lib/cartopy/io/__init__.py index 0a6a2c6dd..958b51100 100644 --- a/lib/cartopy/io/__init__.py +++ b/lib/cartopy/io/__init__.py @@ -13,9 +13,10 @@ import collections from pathlib import Path import string -from urllib.request import urlopen import warnings +from urllib3 import request + from cartopy import config @@ -238,7 +239,7 @@ def _urlopen(self, url): """ warnings.warn(f'Downloading: {url}', DownloadWarning) - return urlopen(url) + return request('GET', url, preload_content=False) @staticmethod def from_config(specification, config_dict=None): diff --git a/lib/cartopy/io/srtm.py b/lib/cartopy/io/srtm.py index 1c88d1de5..17f2fba62 100644 --- a/lib/cartopy/io/srtm.py +++ b/lib/cartopy/io/srtm.py @@ -401,9 +401,9 @@ def _create_srtm_mask(resolution, filename=None): # dependencies of cartopy. from bs4 import BeautifulSoup if filename is None: - from urllib.request import urlopen + from urllib3 import request url = SRTMDownloader._SRTM_BASE_URL.format(resolution=resolution) - with urlopen(url) as f: + with request('GET', url, preload_content=False) as f: html = f.read() else: html = Path(filename).read_text() diff --git a/pyproject.toml b/pyproject.toml index 7018e09eb..1385216d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dependencies = [ "packaging>=21", "pyshp>=2.3", "pyproj>=3.3.1", + "urllib3>=2.3" ] dynamic = ["version"] From 9dfbc049209390fed21576814ef644b4b62ba578 Mon Sep 17 00:00:00 2001 From: rpwagner Date: Fri, 28 Mar 2025 17:11:15 -0700 Subject: [PATCH 2/6] additional urllib3 replacement --- lib/cartopy/io/ogc_clients.py | 4 ++-- lib/cartopy/io/shapereader.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartopy/io/ogc_clients.py b/lib/cartopy/io/ogc_clients.py index 4e463c0b0..a48db7ad2 100644 --- a/lib/cartopy/io/ogc_clients.py +++ b/lib/cartopy/io/ogc_clients.py @@ -19,7 +19,7 @@ import io import math from pathlib import Path -from urllib.parse import urlparse +from urllib3.util import parse_url import warnings import weakref from xml.etree import ElementTree @@ -753,7 +753,7 @@ def __init__(self, service, features, getfeature_extra_kwargs=None): # agroenvgeo.data.inra.fr from full address # http://mapserver.gis.umn.edu/mapserver # or https://agroenvgeo.data.inra.fr:443/geoserver/wfs - self.url = urlparse(service).hostname + self.url = parse_url(service).hostname # WebFeatureService of owslib service = WebFeatureService(service) else: diff --git a/lib/cartopy/io/shapereader.py b/lib/cartopy/io/shapereader.py index 704b29244..d202bce5c 100644 --- a/lib/cartopy/io/shapereader.py +++ b/lib/cartopy/io/shapereader.py @@ -30,7 +30,7 @@ import io import itertools from pathlib import Path -from urllib.error import HTTPError +from urllib3.exceptions import HTTPError import shapefile import shapely.geometry as sgeom From 81f5588d87fd59d8c9bee71fb9c5873751040294 Mon Sep 17 00:00:00 2001 From: rpwagner Date: Fri, 28 Mar 2025 17:30:42 -0700 Subject: [PATCH 3/6] need to release connections --- lib/cartopy/io/__init__.py | 1 + lib/cartopy/io/img_tiles.py | 12 ++++++------ lib/cartopy/io/srtm.py | 3 ++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/cartopy/io/__init__.py b/lib/cartopy/io/__init__.py index 958b51100..81bfa3e5b 100644 --- a/lib/cartopy/io/__init__.py +++ b/lib/cartopy/io/__init__.py @@ -228,6 +228,7 @@ def acquire_resource(self, target_path, format_dict): response = self._urlopen(url) target_path.write_bytes(response.read()) + response.release_conn() return target_path diff --git a/lib/cartopy/io/img_tiles.py b/lib/cartopy/io/img_tiles.py index a9e92af38..882f9d206 100644 --- a/lib/cartopy/io/img_tiles.py +++ b/lib/cartopy/io/img_tiles.py @@ -202,7 +202,8 @@ def _image_url(self, tile): pass def get_image(self, tile): - from urllib.request import HTTPError, Request, URLError, urlopen + from urllib3 import request + from urllib3.exceptions import HTTPError if self.cache_path is not None: filename = "_".join([str(i) for i in tile]) + ".npy" @@ -215,13 +216,12 @@ def get_image(self, tile): else: url = self._image_url(tile) try: - request = Request(url, headers={"User-Agent": self.user_agent}) - fh = urlopen(request) - im_data = io.BytesIO(fh.read()) - fh.close() + r = request('GET', url, headers={"User-Agent": self.user_agent}, preload_content=False) + im_data = io.BytesIO(r.read()) + r.release_conn() img = Image.open(im_data) - except (HTTPError, URLError) as err: + except HTTPError as err: print(err) img = Image.fromarray(np.full((256, 256, 3), (250, 250, 250), dtype=np.uint8)) diff --git a/lib/cartopy/io/srtm.py b/lib/cartopy/io/srtm.py index 17f2fba62..f3a14da51 100644 --- a/lib/cartopy/io/srtm.py +++ b/lib/cartopy/io/srtm.py @@ -403,8 +403,9 @@ def _create_srtm_mask(resolution, filename=None): if filename is None: from urllib3 import request url = SRTMDownloader._SRTM_BASE_URL.format(resolution=resolution) - with request('GET', url, preload_content=False) as f: + with request('GET', url, preload_content=False) as r: html = f.read() + r.release_conn() else: html = Path(filename).read_text() From 96e74c1014757e78c1283119ef154d336c323bc5 Mon Sep 17 00:00:00 2001 From: rpwagner Date: Fri, 28 Mar 2025 19:21:05 -0700 Subject: [PATCH 4/6] fix variable name --- lib/cartopy/io/srtm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartopy/io/srtm.py b/lib/cartopy/io/srtm.py index f3a14da51..0506adccc 100644 --- a/lib/cartopy/io/srtm.py +++ b/lib/cartopy/io/srtm.py @@ -404,7 +404,7 @@ def _create_srtm_mask(resolution, filename=None): from urllib3 import request url = SRTMDownloader._SRTM_BASE_URL.format(resolution=resolution) with request('GET', url, preload_content=False) as r: - html = f.read() + html = r.read() r.release_conn() else: html = Path(filename).read_text() From 868b98598ef2709c7ebcb5c0f8f5d7024dd2a33a Mon Sep 17 00:00:00 2001 From: rpwagner Date: Fri, 28 Mar 2025 19:36:22 -0700 Subject: [PATCH 5/6] fix line length --- lib/cartopy/io/img_tiles.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cartopy/io/img_tiles.py b/lib/cartopy/io/img_tiles.py index 882f9d206..ffa990a9b 100644 --- a/lib/cartopy/io/img_tiles.py +++ b/lib/cartopy/io/img_tiles.py @@ -216,7 +216,8 @@ def get_image(self, tile): else: url = self._image_url(tile) try: - r = request('GET', url, headers={"User-Agent": self.user_agent}, preload_content=False) + r = request('GET', url, headers={"User-Agent": self.user_agent}, + preload_content=False) im_data = io.BytesIO(r.read()) r.release_conn() img = Image.open(im_data) From f43e232565d183f026d0f5a7a05fd72064a729ca Mon Sep 17 00:00:00 2001 From: rpwagner Date: Fri, 28 Mar 2025 19:51:07 -0700 Subject: [PATCH 6/6] fix linter errors --- lib/cartopy/io/ogc_clients.py | 2 +- lib/cartopy/io/shapereader.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartopy/io/ogc_clients.py b/lib/cartopy/io/ogc_clients.py index a48db7ad2..613f06059 100644 --- a/lib/cartopy/io/ogc_clients.py +++ b/lib/cartopy/io/ogc_clients.py @@ -19,7 +19,6 @@ import io import math from pathlib import Path -from urllib3.util import parse_url import warnings import weakref from xml.etree import ElementTree @@ -27,6 +26,7 @@ import numpy as np from PIL import Image import shapely.geometry as sgeom +from urllib3.util import parse_url import cartopy diff --git a/lib/cartopy/io/shapereader.py b/lib/cartopy/io/shapereader.py index d202bce5c..7f426284b 100644 --- a/lib/cartopy/io/shapereader.py +++ b/lib/cartopy/io/shapereader.py @@ -30,10 +30,10 @@ import io import itertools from pathlib import Path -from urllib3.exceptions import HTTPError import shapefile import shapely.geometry as sgeom +from urllib3.exceptions import HTTPError from cartopy import config from cartopy.io import Downloader