From 8ca353352b0ade216d958c1f723b75a85dd83be6 Mon Sep 17 00:00:00 2001 From: stdavis Date: Fri, 7 Aug 2026 09:49:39 -0600 Subject: [PATCH 1/5] chore: fix new ruff errors A recent update to Ruff caused some new errors. This addresses them. --- README.md | 12 ++++---- docs/README.md | 4 +-- docs/examples.py | 3 +- setup.py | 7 ++--- src/palletjack/extract.py | 24 +++++++-------- src/palletjack/load.py | 17 +++++------ src/palletjack/transform.py | 6 ++-- src/palletjack/utils.py | 27 +++++++++-------- tests/test_extract.py | 60 ++++++++++++++++++------------------- tests/test_transform.py | 3 +- tests/test_utils.py | 26 ++++++++-------- 11 files changed, 93 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index 12a8741e..60628b35 100644 --- a/README.md +++ b/README.md @@ -34,20 +34,20 @@ The `arcgis` library does all the heavy lifting for spatial data. If the `arcpy` #: Load the data from a Google Sheet gsheet_extractor = extract.GSheetLoader(path_to_service_account_json) - sheet_df = gsheet_extractor.load_specific_worksheet_into_dataframe(sheet_id, 'title of desired sheet', by_title=True) + sheet_df = gsheet_extractor.load_specific_worksheet_into_dataframe(sheet_id, "title of desired sheet", by_title=True) #: Convert the data to points using lat/long fields, clean for uploading - spatial_df = pd.DataFrame.spatial.from_xy(input_df, x_column='longitude', y_column='latitude') + spatial_df = pd.DataFrame.spatial.from_xy(input_df, x_column="longitude", y_column="latitude") renamed_df = transform.DataCleaning.rename_dataframe_columns_for_agol(spatial_df) - cleaned_df = transform.DataCleaning.switch_to_nullable_int(renamed_df, ['an_int_field_with_null_values']) + cleaned_df = transform.DataCleaning.switch_to_nullable_int(renamed_df, ["an_int_field_with_null_values"]) #: Truncate the existing feature service data and load the new data - gis = arcgis.gis.GIS('my_agol_org_url', 'username', 'super-duper-secure-password') - updater = load.ServiceUpdater(gis, 'feature_service_item_id') + gis = arcgis.gis.GIS("my_agol_org_url", "username", "super-duper-secure-password") + updater = load.ServiceUpdater(gis, "feature_service_item_id") updates = updater.truncate_and_load(cleaned_df) #: It even works with stand-alone tables! - table_updater = load.TableUpdater(gis, 'table_service_item_id', service_type='table') + table_updater = load.TableUpdater(gis, "table_service_item_id", service_type="table") table_updates = table_updater.truncate_and_load(cleaned_df) ``` diff --git a/docs/README.md b/docs/README.md index c5ceb32f..c22874fe 100644 --- a/docs/README.md +++ b/docs/README.md @@ -57,9 +57,9 @@ palletjack takes full advantage of python's built-in [`logging`](https://docs.py The client can get a reference to the palletjack logger and add their handlers, formatters, etc to it alongside its own logger: ```python -myapp_logger = logging.getLogger('my_app') +myapp_logger = logging.getLogger("my_app") myapp_logger.setLevel(logging.INFO) -palletjack_logger = logging.getLogger('palletjack') +palletjack_logger = logging.getLogger("palletjack") palletjack_logger.setLevel(logging.INFO) #: set up handlers and formatters #: ... diff --git a/docs/examples.py b/docs/examples.py index bd75ddad..0f764d79 100644 --- a/docs/examples.py +++ b/docs/examples.py @@ -7,6 +7,7 @@ import arcgis import pandas as pd from arcgis.features import GeoAccessor, GeoSeriesAccessor + from palletjack import extract, load, transform, utils @@ -122,5 +123,5 @@ def download_from_sftp_update_agol_reclassify_map(): #: Try to clean up the tempdir (we don't use a context manager); print any errors as a heads up try: tempdir.cleanup() - except Exception as error: + except OSError as error: print(error) diff --git a/setup.py b/setup.py index 0e054f31..9ecc2fda 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,14 @@ -#!/usr/bin/env python -# -*- encoding: utf-8 -*- """ setup.py A module that installs palletjack as a module """ +import runpy from pathlib import Path from setuptools import find_packages, setup -version = {} -with open("src/palletjack/version.py", encoding="utf-8") as fp: - exec(fp.read(), version) +version = runpy.run_path("src/palletjack/version.py") setup( name="ugrc-palletjack", diff --git a/src/palletjack/extract.py b/src/palletjack/extract.py index a7690bd1..fd84a9ad 100644 --- a/src/palletjack/extract.py +++ b/src/palletjack/extract.py @@ -13,7 +13,7 @@ import warnings from contextlib import contextmanager from dataclasses import dataclass, field -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from io import BytesIO from pathlib import Path from string import Template @@ -251,7 +251,7 @@ def combine_worksheets_into_single_dataframe(self, worksheet_dfs): dataframes = list(worksheet_dfs.values()) #: Make sure all the dataframes have the same columns - if not all([set(dataframes[0].columns) == set(df.columns) for df in dataframes]): + if not all(set(dataframes[0].columns) == set(df.columns) for df in dataframes): raise ValueError("Columns do not match; cannot create multi-index dataframe") self._class_logger.debug("Concatting worksheet dataframes %s into a single dataframe", worksheet_dfs.keys()) @@ -401,7 +401,7 @@ def download_file_from_google_drive(self, sharing_link, join_id, pause=0.0): self._class_logger.debug("Row %s: writing to %s", join_id, out_file_path) self._save_response_content(response, out_file_path) return out_file_path - except Exception as err: + except Exception as err: # noqa: BLE001 - remote download clients expose inconsistent exception types self._class_logger.warning("Row %s: Couldn't download %s", join_id, sharing_link) self._class_logger.warning(err) return None @@ -478,7 +478,7 @@ def download_file_from_google_drive_using_api(self, gsheets_client, sharing_link self._class_logger.debug("Row %s: writing to %s", join_id, out_file_path) utils.retry(self._save_get_media_content, get_media_request, out_file_path) return out_file_path - except Exception as err: + except Exception as err: # noqa: BLE001 - remote download clients expose inconsistent exception types self._class_logger.warning("Row %s: Couldn't download %s", join_id, sharing_link) self._class_logger.warning(err) return None @@ -1100,7 +1100,7 @@ class SalesforceRestLoader: access_token_template = Template("https://$org.my.salesforce.com/services/oauth2/token") access_token_url = "" - access_token = {} + access_token: dict[str, str] org_template = Template("https://$org.my.salesforce.com") org_url = "" @@ -1129,6 +1129,7 @@ def __init__(self, org, credentials, sandbox=False) -> None: sandbox (bool, optional): The credentials for sandboxes are different than API users. Defaults to False if it's not a sandbox instance of Salesforce. """ + self.access_token = {} self.client_secret = credentials.client_secret self.client_id = credentials.client_id @@ -1153,7 +1154,7 @@ def _is_token_valid(self, token: dict[str, str]) -> bool: Returns: bool: true if the token is valid """ - if "issued_at" not in token.keys(): + if "issued_at" not in token: return False issued = timedelta.max @@ -1161,8 +1162,8 @@ def _is_token_valid(self, token: dict[str, str]) -> bool: try: ticks = int(token["issued_at"]) - issued = datetime.fromtimestamp(ticks / 1000) - days_from_today = (datetime.now() - issued).days + issued = datetime.fromtimestamp(ticks / 1000, UTC) + days_from_today = (datetime.now(UTC) - issued).days self._class_logger.debug("Token is %s days old", days_from_today) except ValueError: @@ -1170,7 +1171,7 @@ def _is_token_valid(self, token: dict[str, str]) -> bool: return False - return datetime.now() < (issued + lease) + return datetime.now(UTC) < (issued + lease) def _get_token(self) -> dict[str, str]: """Gets a new Salesforce access token if the current one is expired.""" @@ -1358,9 +1359,8 @@ def get_from_endpoint(self, endpoint: str, params: dict | None = None, expand_ac # Treat that as a single record so we still produce a sensible DataFrame. all_records.append(page_data) else: - raise ValueError( - f"Unexpected JSON type {type(page_data).__name__} from {url} on page {page}; " - "expected list or dict." + raise TypeError( + f"Unexpected JSON type {type(page_data).__name__} from {url} on page {page}; expected list or dict." ) total_pages_header = response.headers.get("X-WP-TotalPages") if not total_pages_header: diff --git a/src/palletjack/load.py b/src/palletjack/load.py index 301e4b5b..40463fa6 100644 --- a/src/palletjack/load.py +++ b/src/palletjack/load.py @@ -6,7 +6,7 @@ import logging import shutil import warnings -from datetime import datetime +from datetime import UTC, datetime from pathlib import Path from typing import Literal @@ -232,7 +232,7 @@ def truncate_and_load(self, dataframe: pd.DataFrame, save_old: bool = False) -> self.index, self.itemid, ) - start = datetime.now() + start = datetime.now(UTC) #: Save the data to disk if desired if save_old: @@ -253,7 +253,7 @@ def truncate_and_load(self, dataframe: pd.DataFrame, save_old: bool = False) -> try: self._class_logger.info("Loading new data...") append_count = self._update_service(gdb_item, upsert=False) - self._class_logger.debug("Total truncate and load time: %s", datetime.now() - start) + self._class_logger.debug("Total truncate and load time: %s", datetime.now(UTC) - start) except Exception: if save_old: self._class_logger.error("Append failed. Data saved to %s", saved_layer_path) @@ -418,7 +418,7 @@ def _upload_gdb(self, zipped_gdb_path: Path) -> Item: item_type=item_type, max_items=1, ) - except Exception as error: + except Exception as error: # noqa: BLE001 - ArcGIS API does not expose a stable exception type self._class_logger.warning(f"Error searching for existing gdb item with title {title}") warnings.warn(repr(error)) @@ -463,14 +463,14 @@ def _cleanup(self, gdb_item: Item | None = None, zipped_gdb_path: Path | None = if gdb_item: try: gdb_item.delete() - except Exception as error: + except Exception as error: # noqa: BLE001 - cleanup must not obscure the primary operation result warnings.warn(f"Error deleting gdb item {gdb_item.id} from AGOL") warnings.warn(repr(error)) if zipped_gdb_path: try: zipped_gdb_path.unlink() - except Exception as error: + except Exception as error: # noqa: BLE001 - cleanup must not obscure the primary operation result warnings.warn(f"Error deleting zipped gdb {zipped_gdb_path}") warnings.warn(repr(error)) @@ -628,7 +628,7 @@ def _add_attachments_by_oid(self, attachment_action_df, attachment_path_field): try: result = self.feature_layer.attachments.add(target_oid, filepath) except Exception: - self._class_logger.error("AGOL error while adding %s to OID %s", filepath, target_oid, exc_info=True) + self._class_logger.exception("AGOL error while adding %s to OID %s", filepath, target_oid) self.failed_dict[target_oid] = ("add", filepath) continue @@ -669,13 +669,12 @@ def _overwrite_attachments_by_oid(self, attachment_action_df, attachment_path_fi try: result = self.feature_layer.attachments.update(target_oid, attachment_id, filepath) except Exception: - self._class_logger.error( + self._class_logger.exception( "AGOL error while overwriting %s (attachment ID %s) on OID %s with %s", old_name, attachment_id, target_oid, filepath, - exc_info=True, ) self.failed_dict[target_oid] = ("update", filepath) continue diff --git a/src/palletjack/transform.py b/src/palletjack/transform.py index 41fb35bc..a47a361d 100644 --- a/src/palletjack/transform.py +++ b/src/palletjack/transform.py @@ -3,7 +3,7 @@ import locale import logging import warnings -from datetime import datetime +from datetime import UTC, datetime import arcgis import pandas as pd @@ -55,7 +55,7 @@ def geocode_dataframe(self, dataframe, street_col, zone_col, wkid, rate_limits=( pd.DataFrame.spatial: Geocoded data as a spatially-enabled DataFrame """ - start = datetime.now() + start = datetime.now(UTC) #: Should this return? Should it raise an error instead? if dataframe.empty: @@ -88,7 +88,7 @@ def geocode_dataframe(self, dataframe, street_col, zone_col, wkid, rate_limits=( spatial_dataframe = pd.DataFrame.spatial.from_xy(pd.DataFrame(new_rows), "x", "y", sr=int(wkid)) - end = datetime.now() + end = datetime.now(UTC) self._class_logger.info("%s Records geocoded in %s", len(spatial_dataframe.index), (end - start)) try: self._class_logger.debug("Average time per record: %s", (end - start) / len(spatial_dataframe.index)) diff --git a/src/palletjack/utils.py b/src/palletjack/utils.py index 9eb42171..8d41c958 100644 --- a/src/palletjack/utils.py +++ b/src/palletjack/utils.py @@ -68,7 +68,7 @@ def _inner_retry(worker_method, *args, **kwargs): tries += 1 return _inner_retry(worker_method, *args, **kwargs) else: - raise error + raise return _inner_retry(worker_method, *args, **kwargs) @@ -158,7 +158,7 @@ def rename_fields(dataframe, field_mapping): pd.DataFrame: Dataframe with renamed fields """ - for original_name in field_mapping.keys(): + for original_name in field_mapping: if original_name not in dataframe.columns: raise ValueError(f"Field {original_name} not found in dataframe.") @@ -200,9 +200,8 @@ def check_field_set_to_unique(featurelayer, field_name): if field_name not in fields: raise RuntimeError(f'{field_name} does not have a "unique constraint" set within the feature layer') for field in featurelayer.properties.indexes: - if field["fields"] == field_name: - if not field["isUnique"]: - raise RuntimeError(f'{field_name} does not have a "unique constraint" set within the feature layer') + if field["fields"] == field_name and not field["isUnique"]: + raise RuntimeError(f'{field_name} does not have a "unique constraint" set within the feature layer') class Geocoding: @@ -233,7 +232,7 @@ def geocode_addr(street, zone, api_key, rate_limits, **api_args): try: geocode_result_dict = retry(Geocoding._geocode_api_call, url, api_args) - except Exception as error: + except Exception as error: # noqa: BLE001 - requests and retry callbacks may raise distinct exception types module_logger.error(error) return (0, 0, 0.0, "No API response") @@ -409,7 +408,7 @@ def convert_to_gdf(dataframe): #: just a normal df, convert to gdf w/o geometry (allows us to write as table to gdb) try: - dataframe.spatial.geometry_type # raises KeyError if this is a regular dataframe + _ = dataframe.spatial.geometry_type # raises KeyError if this is a regular dataframe except KeyError: return gpd.GeoDataFrame(dataframe, geometry=None) @@ -445,7 +444,7 @@ def save_to_gdb(table_or_layer, directory): gdf = gpd.GeoDataFrame(dataframe) out_path = Path(directory, "backup.gdb") - out_layer = f"{table_or_layer.properties.name}_{datetime.date.today().strftime('%Y_%m_%d')}" + out_layer = f"{table_or_layer.properties.name}_{datetime.datetime.now(datetime.UTC).strftime('%Y_%m_%d')}" module_logger.debug("Saving existing data to %s", out_path) try: gdf.to_file(out_path, layer=out_layer, engine="pyogrio", driver="OpenFileGDB") @@ -618,7 +617,7 @@ def _check_geometry_types(self): try: new_geometry_types = self.new_dataframe.spatial.geometry_type - except Exception: #: If it's not an sedf, the call to geometry_type raises a general Exception, so try gdf + except Exception: # noqa: BLE001 - the ArcGIS accessor raises a generic exception for non-SEDF frames new_geometry_types = self._condense_geopandas_multi_types(self.new_dataframe.geom_type.unique()) if len(new_geometry_types) > 1: @@ -809,9 +808,11 @@ def check_for_np_inf(self): non_spatial_columns = [col for col in self.new_dataframe.columns if self.new_dataframe[col].dtype != "geometry"] for column in non_spatial_columns: #: Only check float columns; in our data model, only floating-point fields can meaningfully contain Β±inf - if pd.api.types.is_float_dtype(self.new_dataframe[column].dtype): - if np.isinf(self.new_dataframe[column]).any(): - columns_with_inf.append(column) + if ( + pd.api.types.is_float_dtype(self.new_dataframe[column].dtype) + and np.isinf(self.new_dataframe[column]).any() + ): + columns_with_inf.append(column) if columns_with_inf: warnings.warn( @@ -972,7 +973,7 @@ def _chunk_dataframe(dataframe, chunk_size): ) starts = range(0, df_length, chunk_size) - ends = [start + chunk_size if start + chunk_size < df_length else df_length for start in starts] + ends = [min(df_length, start + chunk_size) for start in starts] list_of_dataframes = [dataframe.iloc[start:end] for start, end in zip(starts, ends)] return list_of_dataframes diff --git a/tests/test_extract.py b/tests/test_extract.py index 821125ee..cca366a1 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -1,8 +1,9 @@ import json import logging import re -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from pathlib import Path +from typing import ClassVar import geodatasets import geopandas as gpd @@ -44,7 +45,7 @@ def test_load_specific_worksheet_into_dataframe_by_title(self, mocker): extract.GSheetLoader.load_specific_worksheet_into_dataframe(gsheet_loader_mock, "foobar", "2015", by_title=True) - sheet_mock.worksheet.assert_not_called + sheet_mock.worksheet.assert_not_called() sheet_mock.worksheet_by_title.assert_called_once_with("2015") def test_load_all_worksheets_into_dataframes_single_worksheet(self, mocker): @@ -818,9 +819,8 @@ def test_sftp_connection_context_manager_connection_failure(self, mocker): mocker.patch("palletjack.extract.paramiko.Transport", return_value=transport_mock) - with pytest.raises(Exception, match="Connection failed"): - with loader._sftp_connection() as sftp: - pass + with pytest.raises(Exception, match="Connection failed"), loader._sftp_connection() as sftp: + pass # Verify cleanup happened even though connection failed # sftp was never created, so it shouldn't be closed @@ -835,9 +835,11 @@ def test_sftp_connection_context_manager_auth_failure(self, mocker): mocker.patch("palletjack.extract.paramiko.Transport", return_value=transport_mock) - with pytest.raises(paramiko.AuthenticationException, match="Authentication failed"): - with loader._sftp_connection() as sftp: - pass + with ( + pytest.raises(paramiko.AuthenticationException, match="Authentication failed"), + loader._sftp_connection() as sftp, + ): + pass # Verify cleanup happened transport_mock.close.assert_called_once() @@ -852,9 +854,8 @@ def test_sftp_connection_context_manager_sftp_client_failure(self, mocker): mocker.patch("palletjack.extract.paramiko.Transport", return_value=transport_mock) - with pytest.raises(Exception, match="SFTP client creation failed"): - with loader._sftp_connection() as sftp: - pass + with pytest.raises(Exception, match="SFTP client creation failed"), loader._sftp_connection() as sftp: + pass # Verify cleanup happened - transport should be closed even though sftp wasn't created transport_mock.close.assert_called_once() @@ -870,10 +871,9 @@ def test_sftp_connection_context_manager_exception_during_operation(self, mocker sftp_class_mock = mocker.patch("palletjack.extract.paramiko.SFTPClient") sftp_class_mock.from_transport.return_value = sftp_mock - with pytest.raises(RuntimeError, match="Operation failed"): - with loader._sftp_connection() as sftp: - # Simulate an exception during file operations - raise RuntimeError("Operation failed") + with pytest.raises(RuntimeError, match="Operation failed"), loader._sftp_connection() as sftp: + # Simulate an exception during file operations + raise RuntimeError("Operation failed") # Verify cleanup happened despite the exception sftp_mock.close.assert_called_once() @@ -978,7 +978,7 @@ class TestRESTServiceLoader: def test_get_features_gets_max_record_count_from_properties(self, mocker): layer_mock = mocker.patch("palletjack.extract.ServiceLayer").return_value layer_mock.max_record_count = 42 - layer_mock.get_object_ids.return_value = list(range(0, 142)) + layer_mock.get_object_ids.return_value = list(range(142)) mocker.patch("palletjack.extract.pd.concat") mocker.patch("palletjack.extract.time.sleep") @@ -986,13 +986,13 @@ def test_get_features_gets_max_record_count_from_properties(self, mocker): extract.RESTServiceLoader.get_features(mocker.Mock(), layer_mock, chunk_size=None) - chunker_mock.assert_called_once_with(list(range(0, 142)), 42) + chunker_mock.assert_called_once_with(list(range(142)), 42) def test_get_features_chunks_smaller_final_chunk(self, mocker): layer_mock = mocker.patch("palletjack.extract.ServiceLayer").return_value layer_mock.max_record_count = 100 layer_mock.oid_field = "OBJECTID" - layer_mock.get_object_ids.return_value = list(range(0, 142)) + layer_mock.get_object_ids.return_value = list(range(142)) mocker.patch("palletjack.extract.pd.concat") mocker.patch("palletjack.extract.time.sleep") @@ -1000,7 +1000,7 @@ def test_get_features_chunks_smaller_final_chunk(self, mocker): extract.RESTServiceLoader.get_features(mocker.Mock(), layer_mock) assert layer_mock.get_unique_id_list_as_dataframe.call_args_list == [ - mocker.call("OBJECTID", list(range(0, 100))), + mocker.call("OBJECTID", list(range(100))), mocker.call("OBJECTID", list(range(100, 142))), ] @@ -1042,7 +1042,7 @@ def test_get_features_concats_with_new_index(self, mocker): layer_mock = mocker.patch("palletjack.extract.ServiceLayer").return_value layer_mock.max_record_count = 100 layer_mock.oid_field = "OBJECTID" - layer_mock.get_object_ids.return_value = list(range(0, 142)) + layer_mock.get_object_ids.return_value = list(range(142)) layer_mock.get_unique_id_list_as_dataframe.side_effect = [ pd.DataFrame(["a", "b", "c"], columns=["SHAPE"]), pd.DataFrame(["d", "e", "f"], columns=["SHAPE"]), @@ -1061,7 +1061,7 @@ def test_get_features_retries_on_failed_oid_get(self, mocker): layer_mock = mocker.patch("palletjack.extract.ServiceLayer").return_value layer_mock.max_record_count = 100 layer_mock.oid_field = "OBJECTID" - layer_mock.get_object_ids.side_effect = [RuntimeError, list(range(0, 142))] + layer_mock.get_object_ids.side_effect = [RuntimeError, list(range(142))] mocker.patch("palletjack.extract.pd.concat") mocker.patch("palletjack.extract.time.sleep") @@ -1558,7 +1558,7 @@ def test_get_object_id_field_uses_OBJECTID_if_no_field(self, mocker): class TestSalesForceLoader: def ticks(self, dt): - return (dt - datetime(1970, 1, 1)).total_seconds() * 1000 + return (dt - datetime(1970, 1, 1, tzinfo=UTC)).total_seconds() * 1000 @pytest.fixture def credentials(self): @@ -1570,7 +1570,7 @@ def credentials(self): def loader(self, credentials): return extract.SalesforceRestLoader(ORG, credentials) - url_template_test_data = [ + url_template_test_data: ClassVar = [ ( True, "https://ugrc.sandbox.my.salesforce.com/services/oauth2/token", @@ -1623,19 +1623,19 @@ def test_is_token_valid_returns_false_if_issued_at_cannot_be_converted_to_number assert not loader._is_token_valid({"issued_at": "not a number"}) def test_is_token_valid_returns_true_if_token_is_valid(self, loader): - issued_at = datetime.now() - timedelta(days=15) + issued_at = datetime.now(UTC) - timedelta(days=15) token = {"issued_at": self.ticks(issued_at)} assert loader._is_token_valid(token) def test_is_token_valid_returns_false_if_token_is_expired(self, loader): - issued_at = datetime.now() - timedelta(days=31) + issued_at = datetime.now(UTC) - timedelta(days=31) token = {"issued_at": self.ticks(issued_at)} assert not loader._is_token_valid(token) def test_get_token_returns_cached_token_if_valid(self, loader): - issued_at = datetime.now() - timedelta(days=15) + issued_at = datetime.now(UTC) - timedelta(days=15) token = {"issued_at": self.ticks(issued_at), "access_token": "token"} loader.access_token = token @@ -1682,7 +1682,7 @@ def test_get_token_uses_sandbox_credentials(self, loader, mocker): ) def test_get_token_returns_new_token_if_cached_token_is_expired(self, loader): - issued_at = datetime.now() - timedelta(days=31) + issued_at = datetime.now(UTC) - timedelta(days=31) token = {"issued_at": self.ticks(issued_at), "access_token": "token"} loader.access_token = token @@ -1692,7 +1692,7 @@ def test_get_token_returns_new_token_if_cached_token_is_expired(self, loader): assert loader._get_token() == {"access_token": "new_token", "issued_at": "now"} def test_query_records_raises_error_on_failed_request(self, loader): - issued_at = datetime.now() - timedelta(days=1) + issued_at = datetime.now(UTC) - timedelta(days=1) token = {"issued_at": self.ticks(issued_at), "access_token": "token"} loader.access_token = token @@ -1866,10 +1866,10 @@ def test_get_from_endpoint_dict_response_treated_as_single_record(self, loader): def test_get_from_endpoint_unexpected_json_type_raises(self, loader): # If the endpoint returns something other than list or dict, a clear - # ValueError should be raised. + # TypeError should be raised. with requests_mock.Mocker() as m: m.get(WP_FULL_URL, json="unexpected string response", headers={"X-WP-TotalPages": "1"}) - with pytest.raises(ValueError, match="Unexpected JSON type"): + with pytest.raises(TypeError, match="Unexpected JSON type"): loader.get_from_endpoint(WP_ENDPOINT) def test_get_from_endpoint_missing_total_pages_header_defaults_to_1(self, loader): diff --git a/tests/test_transform.py b/tests/test_transform.py index cd77970a..764920ac 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -2,11 +2,12 @@ import re import numpy as np -import palletjack import pandas as pd import pytest from pandas import testing as tm +import palletjack + class TestAPIGeocoder: def test_geocode_dataframe_calls_with_right_args(self, mocker): diff --git a/tests/test_utils.py b/tests/test_utils.py index 345a5f75..87329670 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -131,7 +131,7 @@ def test_retry_fails_after_four_failures(self, mocker): ] mocker.patch("palletjack.utils.sleep") - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa: B017 - verifies retry propagates arbitrary callback failures answer = palletjack.utils.retry(mock.function, "a", "b") assert mock.function.call_count == 4 @@ -141,7 +141,7 @@ def test_retry_uses_global_retry_max_value(self, mocker, set_max_tries): mock.function.side_effect = [Exception, Exception, 42] mocker.patch("palletjack.utils.sleep") - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa: B017 - verifies retry propagates arbitrary callback failures answer = palletjack.utils.retry(mock.function, "a", "b") assert mock.function.call_count == 2 @@ -293,9 +293,7 @@ def test_geocode_addr_returns_null_island_on_404(self, mocker, caplog): response_mock.status_code = 404 def bool_mock(self): - if self.status_code < 400: - return True - return False + return self.status_code < 400 response_mock.__bool__ = bool_mock @@ -313,9 +311,7 @@ def test_geocode_addr_404_doesnt_raise_no_response_error(self, mocker, caplog): response_mock.status_code = 404 def bool_mock(self): - if self.status_code < 400: - return True - return False + return self.status_code < 400 response_mock.__bool__ = bool_mock @@ -468,7 +464,7 @@ def test_validate_api_key_handles_network_exception(self, mocker, caplog): req_mock = mocker.patch("palletjack.utils.requests", autospec=True) mocker.patch("palletjack.utils.sleep") # caplog.set_level(logging.DEBUG) - req_mock.get.side_effect = [IOError("Random Error")] * 4 + req_mock.get.side_effect = [OSError("Random Error")] * 4 with pytest.raises( RuntimeError, @@ -1007,13 +1003,13 @@ def test_check_live_and_new_field_types_warns_on_bigint_on_esri_int_field(self, properties_mock = mocker.Mock() properties_mock.fields = [{"name": "ints", "type": "esriFieldTypeInteger"}] + checker = palletjack.utils.FieldChecker(properties_mock, new_df) with pytest.warns( UserWarning, match=re.escape( "Field ints has a source 64bit dtype (int64) which may be incompatible with Esri field type esriFieldTypeInteger." ), ): - checker = palletjack.utils.FieldChecker(properties_mock, new_df) checker.check_live_and_new_field_types_match(["ints"]) def test_check_geometry_types_normal(self, mocker): @@ -1393,7 +1389,9 @@ def test_check_field_length_raises_on_long_string(self, mocker): with pytest.raises(ValueError) as exc_info: checker.check_field_length(["foo"]) - assert "Row 2, column foo length of 27 in new data exceeds the live data max length of 10" in str(exc_info.value) + assert "Row 2, column foo length of 27 in new data exceeds the live data max length of 10" in str( + exc_info.value + ) def test_check_field_length_uses_fields_arg(self, mocker): properties_mock = mocker.Mock() @@ -1877,7 +1875,7 @@ def test_check_delete_oids_are_in_live_data_warns_on_missing_oid(self, mocker): class TestSaveDataframeToGDF: def test_save_to_gdb_calls_to_file_with_right_path(self, mocker): expected_out_path = Path("foo", "backup.gdb") - expected_out_layer = f"flayer_{datetime.date.today().strftime('%Y_%m_%d')}" + expected_out_layer = f"flayer_{datetime.datetime.now(datetime.UTC).strftime('%Y_%m_%d')}" mock_fl = mocker.Mock(spec=FeatureLayer) mock_fl.properties.name = "flayer" @@ -1894,7 +1892,7 @@ def test_save_to_gdb_calls_to_file_with_right_path(self, mocker): def test_save_to_gdb_uses_gdb_for_tables(self, mocker): expected_out_path = Path("foo", "backup.gdb") - expected_out_layer = f"table_{datetime.date.today().strftime('%Y_%m_%d')}" + expected_out_layer = f"table_{datetime.datetime.now(datetime.UTC).strftime('%Y_%m_%d')}" mock_tb = mocker.Mock(spec=Table) mock_tb.properties.name = "table" @@ -1920,7 +1918,7 @@ def test_save_to_gdb_doesnt_save_empty_data(self, mocker): def test_save_to_gdb_raises_on_gdb_write_error(self, mocker): gdb_path = Path("/foo/bar/backup.gdb") - date = datetime.date.today().strftime("%Y_%m_%d") + date = datetime.datetime.now(datetime.UTC).strftime("%Y_%m_%d") expected_error = f"Error writing flayer_{date} to {gdb_path}. Verify {gdb_path.parent} exists and is writable." mock_fl = mocker.Mock(spec=FeatureLayer) From e61f3773c6cd2cd89ba28502adc57fcaebd3c7c7 Mon Sep 17 00:00:00 2001 From: stdavis Date: Fri, 7 Aug 2026 10:01:00 -0600 Subject: [PATCH 2/5] ci: add codecov upload to push workflow So that the PR upload comparison is up-to-date --- .github/actions/test-and-coverage/action.yml | 41 ++++++++++++++++++++ .github/workflows/pull_request.yml | 28 ++----------- .github/workflows/push.yml | 15 +++++++ 3 files changed, 59 insertions(+), 25 deletions(-) create mode 100644 .github/actions/test-and-coverage/action.yml diff --git a/.github/actions/test-and-coverage/action.yml b/.github/actions/test-and-coverage/action.yml new file mode 100644 index 00000000..9c044ea7 --- /dev/null +++ b/.github/actions/test-and-coverage/action.yml @@ -0,0 +1,41 @@ +name: Test and coverage +description: Set up Python, run checks, and upload coverage to Codecov. + +inputs: + codecov-token: + description: Token used to upload coverage to Codecov. + required: true + +runs: + using: composite + steps: + - name: 🐍 Set up Python + uses: actions/setup-python@v6.2.0 + with: + python-version-file: .python-version + cache: pip + cache-dependency-path: setup.py + + - name: πŸ“₯ Install dependencies + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y libkrb5-dev + + - name: πŸ— Install module + shell: bash + run: pip install .[tests] + + - name: 🧢 Lint + shell: bash + run: ruff check --output-format=github . + + - name: πŸ§ͺ Run pytest + shell: bash + run: pytest + + - name: ⬆️ Upload coverage to Codecov + uses: codecov/codecov-action@v7 + with: + token: ${{ inputs.codecov-token }} + files: ./cov.xml diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 0fcc8150..b6db8af0 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -20,29 +20,7 @@ jobs: with: show-progress: false - - name: 🐍 Set up Python - uses: actions/setup-python@v6.2.0 + - name: πŸ§ͺ Test and upload coverage + uses: ./.github/actions/test-and-coverage with: - python-version-file: .python-version - cache: pip - cache-dependency-path: setup.py - - - name: πŸ“₯ Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libkrb5-dev - - - name: πŸ— Install module - run: pip install .[tests] - - - name: 🧢 Lint - run: ruff check --output-format=github . - - - name: πŸ§ͺ Run pytest - run: pytest - - - name: ⬆️ Upload coverage to Codecov - uses: codecov/codecov-action@v7 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./cov.xml + codecov-token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 2a767858..bdb343bf 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -11,6 +11,21 @@ concurrency: cancel-in-progress: true jobs: + update-base-coverage: + name: Update base code coverage + runs-on: ubuntu-latest + + steps: + - name: ⬇️ Set up code + uses: actions/checkout@v7 + with: + show-progress: false + + - name: πŸ§ͺ Test and upload coverage + uses: ./.github/actions/test-and-coverage + with: + codecov-token: ${{ secrets.CODECOV_TOKEN }} + release-please: name: Create release runs-on: ubuntu-latest From 18451fe534705a594338477f04911dbdf1cdfa51 Mon Sep 17 00:00:00 2001 From: stdavis Date: Fri, 7 Aug 2026 10:24:42 -0600 Subject: [PATCH 3/5] chore: setup.py -> pyproject.toml Also add python version requirement --- .github/actions/test-and-coverage/action.yml | 2 +- .github/workflows/release.yml | 2 +- .vscode/settings.json | 15 +++-- README.md | 2 +- pyproject.toml | 55 ++++++++++++++++ setup.py | 69 -------------------- src/palletjack/version.py | 6 -- 7 files changed, 67 insertions(+), 84 deletions(-) delete mode 100644 setup.py delete mode 100644 src/palletjack/version.py diff --git a/.github/actions/test-and-coverage/action.yml b/.github/actions/test-and-coverage/action.yml index 9c044ea7..797d3f28 100644 --- a/.github/actions/test-and-coverage/action.yml +++ b/.github/actions/test-and-coverage/action.yml @@ -14,7 +14,7 @@ runs: with: python-version-file: .python-version cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: πŸ“₯ Install dependencies shell: bash diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ed0d0394..3f912257 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,7 @@ jobs: with: python-version-file: .python-version cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: πŸ“¦ Build package run: pipx run build diff --git a/.vscode/settings.json b/.vscode/settings.json index ba6eb1f2..9c74ba6f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { "cSpell.words": [ + "addopts", "agol", "AGRC", "arcgis", @@ -37,6 +38,7 @@ "Geocoder", "geodatabase", "geodataframe", + "geodatasets", "geojsons", "geopandas", "getbuffer", @@ -54,15 +56,19 @@ "itertuples", "keyscan", "knownhosts", + "libkrb", "localpath", "mapserv", "minval", + "minversion", "nans", "ndarray", "nojekyll", + "norecursedirs", "oids", "Overwriter", "palletjack", + "paramiko", "PARENTOBJECTID", "pdoc", "PGSQL", @@ -74,16 +80,19 @@ "pyogrio", "pypa", "pypi", + "pyproject", "pysftp", "pytest", "reclassifier", "Reproject", "sedf", + "setuptools", "sftploader", "SOQL", "sqlalchemy", "subsetted", "subsetting", + "testpaths", "trycount", "ugrc", "ujson", @@ -95,12 +104,6 @@ "wkid", "yapf" ], - "coverage-gutters.highlightdark": "rgb(61, 153, 112, .05)", - "coverage-gutters.noHighlightDark": "rgb(255, 65, 54, .05)", - "coverage-gutters.partialHighlightDark": "rgb(255, 133, 27, .05)", - "coverage-gutters.showGutterCoverage": true, - "coverage-gutters.showLineCoverage": true, - "coverage-gutters.showRulerCoverage": false, "editor.codeActionsOnSave": { "source.organizeImports": "explicit" }, diff --git a/README.md b/README.md index 60628b35..5be5133a 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Pallet jack: [forklift's](https://www.github.com/agrc/forklift) little brother. ## Dependencies -`palletjack` relies on the dependencies listed in `setup.py`. These are all available on PyPI and can be installed in most environments, including Google Cloud Functions. +`palletjack` relies on the dependencies listed in `pyproject.toml`. These are all available on PyPI and can be installed in most environments, including Google Cloud Functions. The `arcgis` library does all the heavy lifting for spatial data. If the `arcpy` library is not available (such as in a cloud function), it relies on `shapely` for its geometry engine. diff --git a/pyproject.toml b/pyproject.toml index 17a921a9..0346929e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,63 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "ugrc-palletjack" +version = "6.0.4" +description = "Updating AGOL feature services with data from external tables." +readme = { file = "README.md", content-type = "text/markdown" } +requires-python = ">=3.11,<3.15" +authors = [{ name = "Jake Adams, UGRC", email = "jdadams@utah.gov" }] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Topic :: Utilities", +] +keywords = ["gis"] +dependencies = [ + "arcgis>=2.3,<2.4.3", + "geopandas>=0.14,<1.2", + "geodatasets>=2023.12,<=2026.5.1", + "pg8000>=1.29,<1.32", + "psycopg2-binary==2.9.*", + "pygsheets==2.0.*", + "pysftp==0.2.9", + "setuptools>=80,<83", + "paramiko>=3.0,<6.0", + "SQLAlchemy>=1.4,<2.1", +] + +[project.urls] +Homepage = "https://github.com/agrc/palletjack" +"Issue Tracker" = "https://github.com/agrc/palletjack/issues" + +[project.optional-dependencies] +tests = [ + "pdoc3>=0.10,<0.12", + "pytest-cov>=3,<8", + "pytest-instafail~=0.5", + "pytest-mock>=3.10,<3.16", + "pytest-watch~=4.2", + "pytest>=6,<10", + "requests-mock==1.*", + "ruff==0.*", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/palletjack"] + +[tool.hatch.build] +exclude = ["/.github", "/.vscode"] + [tool.ruff] line-length = 120 lint.ignore = ["E501"] + [tool.ruff.lint.per-file-ignores] "tests/*.py" = ["F841"] + [tool.pytest.ini_options] minversion = "6.0" testpaths = ["tests", "src"] diff --git a/setup.py b/setup.py deleted file mode 100644 index 9ecc2fda..00000000 --- a/setup.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -setup.py -A module that installs palletjack as a module -""" - -import runpy -from pathlib import Path - -from setuptools import find_packages, setup - -version = runpy.run_path("src/palletjack/version.py") - -setup( - name="ugrc-palletjack", - version=version["__version__"], - description="Updating AGOL feature services with data from external tables.", - long_description=(Path(__file__).parent / "README.md").read_text(), - long_description_content_type="text/markdown", - author="Jake Adams, UGRC", - author_email="jdadams@utah.gov", - url="https://github.com/agrc/palletjack", - packages=find_packages("src"), - package_dir={"": "src"}, - include_package_data=True, - zip_safe=True, - classifiers=[ - # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Topic :: Utilities", - ], - project_urls={ - "Issue Tracker": "https://github.com/agrc/palletjack/issues", - }, - keywords=["gis"], - install_requires=[ - "arcgis>=2.3,<2.4.3", - "geopandas>=0.14,<1.2", - "geodatasets>=2023.12,<=2026.5.1", - "pg8000>=1.29,<1.32", - "psycopg2-binary==2.9.*", - "pygsheets==2.0.*", - "pysftp==0.2.9", - "setuptools>=80,<83", - "paramiko>=3.0,<6.0", - "SQLAlchemy>=1.4,<2.1", - ], - extras_require={ - "tests": [ - "pdoc3>=0.10,<0.12", - "pytest-cov>=3,<8", - "pytest-instafail~=0.5", - "pytest-mock>=3.10,<3.16", - "pytest-watch~=4.2", - "pytest>=6,<10", - "requests-mock==1.*", - "ruff==0.*", - ] - }, - setup_requires=[ - "pytest-runner", - ], - entry_points={ - "console_scripts": [ - "palletjack = palletjack.example:process", - ] - }, -) diff --git a/src/palletjack/version.py b/src/palletjack/version.py deleted file mode 100644 index ce0ef21d..00000000 --- a/src/palletjack/version.py +++ /dev/null @@ -1,6 +0,0 @@ -"""A single source of truth for the version in a programmatically-accessible variable. -This file must only include the single line of code below -""" - -#: This will be automatically set by the publish action when it is uploaded to pypi; only change this for local dev -__version__ = "6.0.4" #: x-release-please-version From 45be7426e1fdda566d0e1916fb1c12f8e184d254 Mon Sep 17 00:00:00 2001 From: stdavis Date: Fri, 7 Aug 2026 10:32:00 -0600 Subject: [PATCH 4/5] chore: flesh out copilot instructions file --- .github/copilot-instructions.md | 20 ++++++++++++++++++-- .vscode/settings.json | 2 ++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 2b67d85f..c06e56c5 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,5 +1,21 @@ # palletjack - Agent Onboarding Guide -## Commits +## Project Structure -Use conventional commits as outlined here: https://github.com/agrc/release-composite-action/blob/main/README.md#commits +Palletjack updates ArcGIS Online feature services from external data using an extract-transform-load workflow. Keep source-specific loading in `extract.py`, DataFrame cleanup and geocoding in `transform.py`, and ArcGIS service updates in `load.py`. Put shared helpers, such as retry behavior, in `utils.py`. + +Preserve pandas DataFrame interfaces. Use type hints and Google-style docstrings for public APIs. Follow the existing module-level logging pattern: `logger = logging.getLogger(__name__)`. + +## Python Environment and Validation + +Use the Conda environment named `palletjack` for all Python commands, tests, and build validation. Python 3.11 through 3.14 are supported. + +Run focused tests for changed behavior with `pytest tests/test_.py`, then run `pytest` when the change affects shared or public behavior. Run `ruff check .` before completion. Do not introduce unconfigured formatters or type checkers. + +Tests use `pytest-mock`; prefer `mocker.patch(..., autospec=True)` where practical. Use standard assertions and `pandas.testing` helpers when comparing DataFrames. + +## Documentation and Commits + +Keep public documentation in Google-style docstrings, which are rendered with pdoc3. Update README or docs examples when changing the public API. + +Use conventional commits as outlined here: https://github.com/agrc/release-composite-action/blob/main/README.md#commits. Use a relevant scope when helpful, such as `fix(extract):`, `feat(load):`, or `docs:`. diff --git a/.vscode/settings.json b/.vscode/settings.json index 9c74ba6f..b1adba80 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,6 +9,7 @@ "astype", "authed", "auths", + "autospec", "caplog", "casefolded", "castable", @@ -97,6 +98,7 @@ "ugrc", "ujson", "unclassed", + "unconfigured", "upserted", "upserting", "upserts", From e660de334243b918017d3c74eee66246979afbb1 Mon Sep 17 00:00:00 2001 From: stdavis Date: Fri, 7 Aug 2026 12:38:33 -0600 Subject: [PATCH 5/5] chore: remove Jake --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0346929e..4339f662 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ version = "6.0.4" description = "Updating AGOL feature services with data from external tables." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11,<3.15" -authors = [{ name = "Jake Adams, UGRC", email = "jdadams@utah.gov" }] +authors = [{ name = "UGRC Developers", email = "ugrc-developers@utah.gov" }] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers",