From e9ffed433432ffec668176b1b5e1906230f58548 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Thu, 10 Oct 2024 20:36:32 -0400 Subject: [PATCH 01/32] Update collections.py added from_featureclass --- geostructures/collections.py | 57 ++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/geostructures/collections.py b/geostructures/collections.py index 992fb83..3948672 100644 --- a/geostructures/collections.py +++ b/geostructures/collections.py @@ -136,6 +136,63 @@ def from_fastkml_folder(cls, folder): return FeatureCollection(parse_fastkml(folder)) + @classmethod + def from_featureclass( + cls, + feature_class_path: str, + geometry_type: geoshape = GeoPoint, + time_field: Optional[str] = None + ): + """ + Creates a FeatureCollection from a feature class. + + Args: + - feature_class_path: str + The path to the feature class (can be a file geodatabase or shapefile). + - geometry_type: GeoShape subclass + The type of geometries to use (default is GeoPoint). + - time_field: str, optional + The name of the field containing time data. + + Returns: + - FeatureCollection instance + """ + + # Load the feature class into a Spatially Enabled DataFrame (SEDF) + sedf = pd.DataFrame.spatial.from_featureclass(feature_class_path) + sedf['SHAPE'] = sedf['SHAPE'].apply(lambda x: shape(x.__geo_interface__)) + sedf = sedf.rename(columns={'SHAPE': 'geometry'}) + + # Handle time field + if time_field: + if time_field in sedf.columns: + # Check if the time_field is already in datetime format + if not pd.api.types.is_datetime64_any_dtype(sedf[time_field]): + sedf[time_field] = pd.to_datetime(sedf[time_field], errors='coerce') + time_values = sedf[time_field].tolist() + else: + time_values = None + raise ValueError(f"Time field '{time_field}' not found in the feature class.") + else: + time_values = None + + # Create the FeatureCollection using from_geopandas + feature_collection = cls(geometry_type).from_geopandas(sedf) + + # Assign the 'dt' attribute if time_field is provided + if time_values: + for feature, dt_value in zip(feature_collection, time_values): + if pd.notnull(dt_value): + # Convert pandas Timestamp to native datetime if necessary + if isinstance(dt_value, pd.Timestamp): + feature.dt = TimeInterval(dt_value.to_pydatetime(), dt_value.to_pydatetime()) + else: + feature.dt = TimeInterval(dt_value, dt_value) + else: + feature.dt = None # Handle missing time values if necessary + + return feature_collection + @classmethod def from_geojson( cls, From 68d9b39d57c9fc4bc3bfd696ec43f8dd3dc44922 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Thu, 10 Oct 2024 21:09:23 -0400 Subject: [PATCH 02/32] Update collections.py to_featureclass --- geostructures/collections.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/geostructures/collections.py b/geostructures/collections.py index 3948672..ad8cfaa 100644 --- a/geostructures/collections.py +++ b/geostructures/collections.py @@ -13,7 +13,10 @@ from typing import cast, Any, List, Dict, Optional, Union, Tuple, TypeVar from zipfile import ZipFile +from arcgis.features import GeoAcessor import numpy as np +import pandas as pd +from shapely import geometry from geostructures import Coordinate, LOGGER from geostructures._base import PolygonLikeMixin, PointLikeMixin, LineLikeMixin, MultiShapeBase, BaseShape @@ -462,6 +465,11 @@ def to_fastkml_folder(self, folder_name: str): features=[x.to_fastkml_placemark() for x in self.geoshapes] ) + def to_featureclass(self, geodatabase, filename): + gdf = self.to_geopandas() + sedf = GeoAccessor.from_geodataframe(gdf) + sedf.spatial.to_featureclass(f'{geodatabase}\{filename}') + def to_geojson(self, properties: Optional[Dict] = None, **kwargs): return { 'type': 'FeatureCollection', From 4120510426a50f2c1c9068788d63e9314f449060 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Thu, 10 Oct 2024 21:26:38 -0400 Subject: [PATCH 03/32] Update collections.py --- geostructures/collections.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/geostructures/collections.py b/geostructures/collections.py index ad8cfaa..101f685 100644 --- a/geostructures/collections.py +++ b/geostructures/collections.py @@ -13,10 +13,7 @@ from typing import cast, Any, List, Dict, Optional, Union, Tuple, TypeVar from zipfile import ZipFile -from arcgis.features import GeoAcessor import numpy as np -import pandas as pd -from shapely import geometry from geostructures import Coordinate, LOGGER from geostructures._base import PolygonLikeMixin, PointLikeMixin, LineLikeMixin, MultiShapeBase, BaseShape @@ -141,9 +138,9 @@ def from_fastkml_folder(cls, folder): @classmethod def from_featureclass( - cls, - feature_class_path: str, - geometry_type: geoshape = GeoPoint, + cls, + feature_class_path: str, + geometry_type: BaseShape = GeoPoint, time_field: Optional[str] = None ): """ @@ -160,6 +157,9 @@ def from_featureclass( Returns: - FeatureCollection instance """ + from arcgis.features import GeoAcessor # noqa: F401 + import pandas as pd + from shapely.geometry import shape # Load the feature class into a Spatially Enabled DataFrame (SEDF) sedf = pd.DataFrame.spatial.from_featureclass(feature_class_path) @@ -466,9 +466,11 @@ def to_fastkml_folder(self, folder_name: str): ) def to_featureclass(self, geodatabase, filename): + from arcgis.features import GeoAcceessor + gdf = self.to_geopandas() sedf = GeoAccessor.from_geodataframe(gdf) - sedf.spatial.to_featureclass(f'{geodatabase}\{filename}') + sedf.spatial.to_featureclass(f'{geodatabase}\\{filename}') def to_geojson(self, properties: Optional[Dict] = None, **kwargs): return { From 1c338b72b6233873663251fcdcd186a383dce523 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Thu, 10 Oct 2024 21:29:23 -0400 Subject: [PATCH 04/32] Update collections.py --- geostructures/collections.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geostructures/collections.py b/geostructures/collections.py index 101f685..903274d 100644 --- a/geostructures/collections.py +++ b/geostructures/collections.py @@ -157,7 +157,7 @@ def from_featureclass( Returns: - FeatureCollection instance """ - from arcgis.features import GeoAcessor # noqa: F401 + from arcgis.features import GeoAcessor # noqa: F401 import pandas as pd from shapely.geometry import shape @@ -466,8 +466,8 @@ def to_fastkml_folder(self, folder_name: str): ) def to_featureclass(self, geodatabase, filename): - from arcgis.features import GeoAcceessor - + from arcgis.features import GeoAccessor + gdf = self.to_geopandas() sedf = GeoAccessor.from_geodataframe(gdf) sedf.spatial.to_featureclass(f'{geodatabase}\\{filename}') From af836010bf8779f09f40e1938229c2837b303ad9 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Fri, 18 Oct 2024 12:28:19 -0400 Subject: [PATCH 05/32] _get_datetime_pandas added internal function to convert timestamp columns --- geostructures/parsers.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/geostructures/parsers.py b/geostructures/parsers.py index fd079ba..d836a44 100644 --- a/geostructures/parsers.py +++ b/geostructures/parsers.py @@ -23,6 +23,27 @@ 'MULTIPOLYGON': MultiGeoPolygon, } +def _get_datetime_pandas(start_time, end_time): + """ + Converts pandas Timestamps to Python datetime objects and returns a TimeInterval. + + Args: + start_time (pd.Timestamp or None): The start time. + end_time (pd.Timestamp or None): The end time. + + Returns: + TimeInterval: The time interval representing the start and end time. + """ + import pandas as pd + + if pd.notnull(start_time) or pd.notnull(end_time): + if isinstance(start_time, pd.Timestamp): + start_time = start_time.to_pydatetime() + + if isinstance(end_time, pd.Timestamp): + end_time = end_time.to_pydatetime() + + return TimeInterval(start_time, end_time) def parse_fastkml( kml, From 82058b814d364ac1bb41a07a01821462abe8567e Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Fri, 18 Oct 2024 12:30:04 -0400 Subject: [PATCH 06/32] parse_arcgis_featureclass Function to parse a featureclass using arcgis package --- geostructures/parsers.py | 100 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/geostructures/parsers.py b/geostructures/parsers.py index d836a44..a493996 100644 --- a/geostructures/parsers.py +++ b/geostructures/parsers.py @@ -45,6 +45,106 @@ def _get_datetime_pandas(start_time, end_time): return TimeInterval(start_time, end_time) +def parse_arcgis_featureclass( + sedf, + time_start_property: Optional[str] = None, + time_end_property: Optional[str] = None, + _shapes: Optional[List[GeoShape]] = None, + _props: Optional[Dict[str, str]] = None, +): + """ + Parses a Spatially Enabled DataFrame (SEDF) from ArcGIS and converts it into geostructures. + + Args: + sedf (DataFrame): + A Spatially Enabled DataFrame containing feature class data. + + time_start_property (str, optional): + The name of the field containing the start time data. + + time_end_property (str, optional): + The name of the field containing the end time data. + + _shapes (List[GeoShape], optional): + Internal use only. Mutated with geostructures as they're extracted from the feature class. + + _props (Dict[str, str], optional): + Internal use only. Information about higher-level containers + to store as properties on the shape for traceability. + + Returns: + List[GeoShape]: A list of GeoShape objects parsed from the feature class. + """ + if _shapes is None: + _shapes = [] + if _props is None: + _props = {} + + property_columns = [col for col in sedf.columns if col != 'SHAPE'] + + for row in sedf.itertuples(): + geometry = row.SHAPE + geometry_type_str = type(geometry).__name__.upper() + if geometry_type_str not in _PARSER_MAP: + raise ValueError(f"Unsupported geometry type: {geometry_type_str}") + + parser = _PARSER_MAP[geometry_type_str] + properties = {col: getattr(row, col) for col in property_columns} + + if geometry_type_str == 'POINT': + lon, lat = geometry.x, geometry.y + shape = parser(Coordinate(lon, lat), properties=properties) + + elif geometry_type_str == 'MULTIPOINT': + # Handle multipoint geometry + points = [Coordinate(point[0], point[1]) for point in geometry['points']] + shape = parser(points, properties=properties) + + elif geometry_type_str == 'POLYLINE': + # Extract parts of the polyline and convert to list of Coordinates + lines = [] + for part in geometry['paths']: + line = [Coordinate(point[0], point[1]) for point in part] + lines.append(line) + + # Create the GeoLineString using the entire line + if len(lines) == 1: + shape = parser(lines[0], properties=properties) + else: + shape = MultiGeoLineString(lines, properties=properties) + + elif geometry_type_str == 'POLYGON': + # Handle MultiPolygon and single Polygon geometries + rings = [] + for ring in geometry['rings']: + # Convert each point in the ring to a Coordinate object + outline = [Coordinate(point[0], point[1]) for point in ring] + # Ensure the outline is closed (first point equals the last point) + if outline[0] != outline[-1]: + outline.append(outline[0]) + rings.append(outline) + + if len(rings) == 1: + # Create a GeoPolygon if there's only one part + outline = rings[0] + holes = rings[1:] if len(rings) > 1 else None + shape = parser(outline, holes=holes, properties=properties) + else: + # Create a MultiGeoPolygon if there are multiple parts + shape = MultiGeoPolygon([parser(ring) for ring in rings], properties=properties) + + else: + raise TypeError(f'Parser for {geometry_type_str} is not available at this time.') + + # Assign the 'dt' attribute if time fields are provided + start_time = getattr(row, time_start_property, None) if isinstance(time_start_property, str) else None + end_time = getattr(row, time_end_property, None) if isinstance(time_end_property, str) else None + shape.dt = _get_datetime_pandas(start_time, end_time) + shape._properties.update(_props) + _shapes.append(shape) + + return _shapes + def parse_fastkml( kml, _shapes: Optional[List[GeoShape]] = None, From b062d08f1932869be49e3916830c10371a946b7b Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Fri, 18 Oct 2024 12:31:37 -0400 Subject: [PATCH 07/32] parse_arcpy_featureclass Added function to parse featureclasses using arcpy --- geostructures/parsers.py | 110 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/geostructures/parsers.py b/geostructures/parsers.py index a493996..7d374f9 100644 --- a/geostructures/parsers.py +++ b/geostructures/parsers.py @@ -145,6 +145,116 @@ def parse_arcgis_featureclass( return _shapes +def parse_arcpy_featureclass( + cursor, + fields: Optional[List[str]] = None, + time_start_property: Optional[str] = None, + time_end_property: Optional[str] = None, + _shapes: Optional[List[GeoShape]] = None, + _props: Optional[Dict[str, str]] = None, +): + """ + Parses feature class data using arcpy and converts it into geostructures. + + Args: + cursor (arcpy.da.SearchCursor): + An arcpy.da.SearchCursor containing the feature class data. + + fields (List[str]): + List of field names to retrieve from the cursor. + + time_start_property (str, optional): + The name of the field containing the start time data. + + time_end_property (str, optional): + The name of the field containing the end time data. + + _shapes (List[GeoShape], optional): + Internal use only. Mutated with geostructures as they're extracted from the feature class. + + _props (Dict[str, str], optional): + Internal use only. Information about higher-level containers + to store as properties on the shape for traceability. + + Returns: + List[GeoShape]: A list of GeoShape objects parsed from the feature class. + """ + if _shapes is None: + _shapes = [] + if _props is None: + _props = {} + + geometry_index = fields.index('SHAPE@') + time_start_index = fields.index(time_start_property) if time_start_property in fields else None + time_end_index = fields.index(time_end_property) if time_end_property in fields else None + cursor.reset() + + for row in cursor: + geometry = row[geometry_index] + geometry_type_str = type(geometry).__name__.upper() + + if geometry_type_str not in _PARSER_MAP: + raise ValueError(f"Unsupported geometry type: {geometry_type_str}") + + parser = _PARSER_MAP[geometry_type_str] + properties = dict(zip(fields, row)) + del properties['SHAPE@'] + + if geometry_type_str == 'POINT': + lon, lat = geometry.X, geometry.Y + shape = parser(Coordinate(lon, lat), properties=properties) + + elif geometry_type_str == 'MULTIPOINT': + # Handle multipoint geometry + points = [Coordinate(point.X, point.Y) for point in geometry] + shape = parser(points, properties=properties) + + elif geometry_type_str == 'POLYLINE': + # Parse the polyline geometry from the ESRI format to GeoLineString + lines = [] + for part in geometry.getPart(): + # Convert each point in the part to a Coordinate object + line = [Coordinate(point.X, point.Y) for point in part] + lines.append(line) + + # Create the GeoLineString using the entire line + if len(lines) == 1: + shape = parser(lines[0], properties=properties) + else: + shape = MultiGeoLineString(lines, properties=properties) + + elif geometry_type_str == 'POLYGON': + # Parse the polygon geometry from the ESRI format to GeoPolygon or MultiGeoPolygon + rings = [] + for part in geometry.getPart(): + # Convert each point in the ring to a Coordinate object + outline = [Coordinate(point.X, point.Y) for point in part] + # Ensure the outline is closed (first point equals the last point) + if outline[0] != outline[-1]: + outline.append(outline[0]) + rings.append(outline) + + if len(rings) == 1: + # Create a GeoPolygon if there's only one part + outline = rings[0] + holes = rings[1:] if len(rings) > 1 else None + shape = parser(outline, holes=holes, properties=properties) + else: + # Create a MultiGeoPolygon if there are multiple parts + shape = MultiGeoPolygon([parser(ring) for ring in rings], properties=properties) + + else: + raise TypeError(f'Parser for {geometry_type_str} not available at this time.') + + # Assign the 'dt' attribute if time_start_property or time_end_property is provided and exists in the row + start_time = row[time_start_index] if isinstance(time_start_property, str) and time_start_index is not None else None + end_time = row[time_end_index] if isinstance(time_end_property, str) and time_end_index is not None else None + shape.dt = _get_datetime_pandas(start_time, end_time) + shape._properties.update(_props) + _shapes.append(shape) + + return _shapes + def parse_fastkml( kml, _shapes: Optional[List[GeoShape]] = None, From c1b1fae15564ad8daf67e0b9440da0bd195259e8 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Fri, 18 Oct 2024 12:39:33 -0400 Subject: [PATCH 08/32] class method for from_arcgis_featureclass converts a featureclass to a Track or FeatureCollection by creating a Spatial Enabled DataFrame (SEDF) and parsing it into GeoShapes --- geostructures/collections.py | 61 ++++++++++-------------------------- 1 file changed, 17 insertions(+), 44 deletions(-) diff --git a/geostructures/collections.py b/geostructures/collections.py index 903274d..7843f72 100644 --- a/geostructures/collections.py +++ b/geostructures/collections.py @@ -137,64 +137,37 @@ def from_fastkml_folder(cls, folder): return FeatureCollection(parse_fastkml(folder)) @classmethod - def from_featureclass( + def from_arcgis_featureclass( cls, feature_class_path: str, - geometry_type: BaseShape = GeoPoint, - time_field: Optional[str] = None + time_start_property: Optional[str] = None, + time_end_property: Optional[str] = None ): """ - Creates a FeatureCollection from a feature class. + Creates an instance of the class from an ArcGIS feature class. Args: - - feature_class_path: str - The path to the feature class (can be a file geodatabase or shapefile). - - geometry_type: GeoShape subclass - The type of geometries to use (default is GeoPoint). - - time_field: str, optional - The name of the field containing time data. + feature_class_path (str): + The path to the feature class. + time_start_property (Optional[str]): + The name of the field containing the start time data. Defaults to None. + time_end_property (Optional[str]): + The name of the field containing the end time data. Defaults to None. Returns: - - FeatureCollection instance + An instance of the class populated with geoshapes parsed from the feature class. """ from arcgis.features import GeoAcessor # noqa: F401 import pandas as pd - from shapely.geometry import shape - # Load the feature class into a Spatially Enabled DataFrame (SEDF) + # Convert the feature class into a Spatially Enabled DataFrame (SEDF) for further processing sedf = pd.DataFrame.spatial.from_featureclass(feature_class_path) - sedf['SHAPE'] = sedf['SHAPE'].apply(lambda x: shape(x.__geo_interface__)) - sedf = sedf.rename(columns={'SHAPE': 'geometry'}) - - # Handle time field - if time_field: - if time_field in sedf.columns: - # Check if the time_field is already in datetime format - if not pd.api.types.is_datetime64_any_dtype(sedf[time_field]): - sedf[time_field] = pd.to_datetime(sedf[time_field], errors='coerce') - time_values = sedf[time_field].tolist() - else: - time_values = None - raise ValueError(f"Time field '{time_field}' not found in the feature class.") - else: - time_values = None - - # Create the FeatureCollection using from_geopandas - feature_collection = cls(geometry_type).from_geopandas(sedf) - - # Assign the 'dt' attribute if time_field is provided - if time_values: - for feature, dt_value in zip(feature_collection, time_values): - if pd.notnull(dt_value): - # Convert pandas Timestamp to native datetime if necessary - if isinstance(dt_value, pd.Timestamp): - feature.dt = TimeInterval(dt_value.to_pydatetime(), dt_value.to_pydatetime()) - else: - feature.dt = TimeInterval(dt_value, dt_value) - else: - feature.dt = None # Handle missing time values if necessary - return feature_collection + # Parse the SEDF to extract geoshapes and convert them into the desired format + geoshapes = parse_arcgis_featureclass(sedf, time_start_property, time_end_property) + + # Return an instance of the class with the parsed geoshapes + return cls(geoshapes) @classmethod def from_geojson( From 42903a09b49f083eb37b7e10e76917f71fb81cb0 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Fri, 18 Oct 2024 12:48:05 -0400 Subject: [PATCH 09/32] class method for from_arcpy_featureclass method to convert a feature class to Track or FeatureCollection using arcpy SearchCursor and featureclass parser --- geostructures/collections.py | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/geostructures/collections.py b/geostructures/collections.py index 7843f72..e91bfe0 100644 --- a/geostructures/collections.py +++ b/geostructures/collections.py @@ -169,6 +169,46 @@ def from_arcgis_featureclass( # Return an instance of the class with the parsed geoshapes return cls(geoshapes) + @classmethod + def from_arcpy_featureclass( + cls, + feature_class_path: str, + time_start_property: Optional[str] = None, + time_end_property: Optional[str] = None + ): + """ + Creates an instance of the class from an ArcGIS feature class. + + Args: + feature_class_path (str): + The path to the feature class. + time_start_property (Optional[str]): + The name of the field containing the start time data. Defaults to None. + time_end_property (Optional[str]): + The name of the field containing the end time data. Defaults to None. + + Returns: + An instance of the class populated with geoshapes parsed from the feature class. + """ + import arcpy + + # Create a set for time properties to ensure uniqueness + time_properties = {time_start_property, time_end_property} if time_start_property and time_end_property else set() + + # Initialize fields list with 'SHAPE@' and add unique time properties + fields = ['SHAPE@'] + list(time_properties) + + # Add all other fields from the feature class, ensuring no duplicates + fields.extend([f.name for f in arcpy.ListFields(feature_class_path) if f.name not in fields]) + + # Use an arcpy SearchCursor to iterate over the feature class rows and extract the relevant fields + with arcpy.da.SearchCursor(feature_class_path, fields) as cursor: + # Parse the feature class using the cursor and provided fields + geoshapes = parser_arcpy_featureclass(cursor, fields, time_start_property, time_end_property) + + # Return an instance of the class with the parsed geoshapes + return cls(geoshapes) + @classmethod def from_geojson( cls, From bca3b69939f3e237470a86235d3692b249d038d3 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Fri, 18 Oct 2024 12:55:24 -0400 Subject: [PATCH 10/32] Update collections.py --- geostructures/collections.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/geostructures/collections.py b/geostructures/collections.py index e91bfe0..c526f4b 100644 --- a/geostructures/collections.py +++ b/geostructures/collections.py @@ -147,11 +147,11 @@ def from_arcgis_featureclass( Creates an instance of the class from an ArcGIS feature class. Args: - feature_class_path (str): + feature_class_path (str): The path to the feature class. - time_start_property (Optional[str]): + time_start_property (Optional[str]): The name of the field containing the start time data. Defaults to None. - time_end_property (Optional[str]): + time_end_property (Optional[str]): The name of the field containing the end time data. Defaults to None. Returns: @@ -159,6 +159,7 @@ def from_arcgis_featureclass( """ from arcgis.features import GeoAcessor # noqa: F401 import pandas as pd + from geostructures.parsers import parse_arcgis_featureclass # Convert the feature class into a Spatially Enabled DataFrame (SEDF) for further processing sedf = pd.DataFrame.spatial.from_featureclass(feature_class_path) @@ -180,20 +181,23 @@ def from_arcpy_featureclass( Creates an instance of the class from an ArcGIS feature class. Args: - feature_class_path (str): + feature_class_path (str): The path to the feature class. - time_start_property (Optional[str]): + time_start_property (Optional[str]): The name of the field containing the start time data. Defaults to None. - time_end_property (Optional[str]): + time_end_property (Optional[str]): The name of the field containing the end time data. Defaults to None. Returns: An instance of the class populated with geoshapes parsed from the feature class. """ import arcpy + from geostructures.parsers import parse_arcpy_featureclass # Create a set for time properties to ensure uniqueness - time_properties = {time_start_property, time_end_property} if time_start_property and time_end_property else set() + time_properties = {time_start_property, time_end_property} if + time_start_property and time_end_property else + set() # Initialize fields list with 'SHAPE@' and add unique time properties fields = ['SHAPE@'] + list(time_properties) From b033fc30c70ae97f1e0af70c2a1eac4f469e8963 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Fri, 18 Oct 2024 12:59:51 -0400 Subject: [PATCH 11/32] Update parsers.py --- geostructures/parsers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/geostructures/parsers.py b/geostructures/parsers.py index 7d374f9..1e6d125 100644 --- a/geostructures/parsers.py +++ b/geostructures/parsers.py @@ -9,6 +9,7 @@ from typing import cast, Any, Dict, List, Optional, Union from geostructures.collections import FeatureCollection +from geostructures.coordinates import Coordinate from geostructures.structures import GeoPolygon, GeoPoint, GeoLineString from geostructures.multistructures import MultiGeoPoint, MultiGeoPolygon, MultiGeoLineString from geostructures.typing import GeoShape, SimpleShape @@ -23,6 +24,7 @@ 'MULTIPOLYGON': MultiGeoPolygon, } + def _get_datetime_pandas(start_time, end_time): """ Converts pandas Timestamps to Python datetime objects and returns a TimeInterval. @@ -35,6 +37,7 @@ def _get_datetime_pandas(start_time, end_time): TimeInterval: The time interval representing the start and end time. """ import pandas as pd + from geostructures.time import TimeInterval if pd.notnull(start_time) or pd.notnull(end_time): if isinstance(start_time, pd.Timestamp): @@ -145,6 +148,7 @@ def parse_arcgis_featureclass( return _shapes + def parse_arcpy_featureclass( cursor, fields: Optional[List[str]] = None, @@ -255,6 +259,7 @@ def parse_arcpy_featureclass( return _shapes + def parse_fastkml( kml, _shapes: Optional[List[GeoShape]] = None, From 91570d63e0dd7961d3c6220dfce7d49d8350ed0f Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Fri, 18 Oct 2024 13:03:56 -0400 Subject: [PATCH 12/32] Update collections.py --- geostructures/collections.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/geostructures/collections.py b/geostructures/collections.py index c526f4b..f94e333 100644 --- a/geostructures/collections.py +++ b/geostructures/collections.py @@ -195,9 +195,11 @@ def from_arcpy_featureclass( from geostructures.parsers import parse_arcpy_featureclass # Create a set for time properties to ensure uniqueness - time_properties = {time_start_property, time_end_property} if - time_start_property and time_end_property else - set() + time_properties = set() + if time_start_property: + time_properties.add(time_start_property) + if time_end_property: + time_properties.add(time_end_property) # Initialize fields list with 'SHAPE@' and add unique time properties fields = ['SHAPE@'] + list(time_properties) From dae94911be8486b21ae0588b0129254b97d038a5 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Fri, 18 Oct 2024 13:07:12 -0400 Subject: [PATCH 13/32] Update parsers.py --- geostructures/parsers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/geostructures/parsers.py b/geostructures/parsers.py index 1e6d125..6096320 100644 --- a/geostructures/parsers.py +++ b/geostructures/parsers.py @@ -48,6 +48,7 @@ def _get_datetime_pandas(start_time, end_time): return TimeInterval(start_time, end_time) + def parse_arcgis_featureclass( sedf, time_start_property: Optional[str] = None, From 28ca4298851ada1cdb1710517b1e3eb17b377c40 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:35:30 -0400 Subject: [PATCH 14/32] Update parsers.py --- geostructures/parsers.py | 280 ++++++++++++++------------------------- 1 file changed, 97 insertions(+), 183 deletions(-) diff --git a/geostructures/parsers.py b/geostructures/parsers.py index 6096320..6496fe5 100644 --- a/geostructures/parsers.py +++ b/geostructures/parsers.py @@ -17,6 +17,7 @@ _PARSER_MAP: Dict[str, SimpleShape] = { 'POINT': GeoPoint, + 'POINTGEOMETRY': GeoPoint, 'LINESTRING': GeoLineString, 'POLYGON': GeoPolygon, 'MULTIPOINT': MultiGeoPoint, @@ -50,215 +51,128 @@ def _get_datetime_pandas(start_time, end_time): def parse_arcgis_featureclass( - sedf, - time_start_property: Optional[str] = None, - time_end_property: Optional[str] = None, - _shapes: Optional[List[GeoShape]] = None, - _props: Optional[Dict[str, str]] = None, -): + row, + columns, + time_start_property: Optional[Union[str, datetime]] = None, + time_end_property: Optional[Union[str, datetime]] = None, + time_fmt: Optional[Union[str, List[str]]] = None, +): -> GeoShape """ - Parses a Spatially Enabled DataFrame (SEDF) from ArcGIS and converts it into geostructures. + Parses an ArcGIS feature class row into a geospatial structure. Args: - sedf (DataFrame): - A Spatially Enabled DataFrame containing feature class data. + row: + The row from an ArcGIS feature class, typically obtained using an arcgis GeoAccessor. - time_start_property (str, optional): - The name of the field containing the start time data. + columns: + List of column names to extract attribute values from the row. - time_end_property (str, optional): - The name of the field containing the end time data. + time_start_property: + Optional; the column name or datetime representing the start time. - _shapes (List[GeoShape], optional): - Internal use only. Mutated with geostructures as they're extracted from the feature class. + time_end_property: + Optional; the column name or datetime representing the end time. - _props (Dict[str, str], optional): - Internal use only. Information about higher-level containers - to store as properties on the shape for traceability. + time_fmt: + Optional; the format or list of formats for parsing time strings. Returns: - List[GeoShape]: A list of GeoShape objects parsed from the feature class. + A geospatial object parsed from the feature class row, with attributes and time interval. + + Raises: + ValueError: If the geometry type is not supported. """ - if _shapes is None: - _shapes = [] - if _props is None: - _props = {} - - property_columns = [col for col in sedf.columns if col != 'SHAPE'] - - for row in sedf.itertuples(): - geometry = row.SHAPE - geometry_type_str = type(geometry).__name__.upper() - if geometry_type_str not in _PARSER_MAP: - raise ValueError(f"Unsupported geometry type: {geometry_type_str}") - - parser = _PARSER_MAP[geometry_type_str] - properties = {col: getattr(row, col) for col in property_columns} - - if geometry_type_str == 'POINT': - lon, lat = geometry.x, geometry.y - shape = parser(Coordinate(lon, lat), properties=properties) - - elif geometry_type_str == 'MULTIPOINT': - # Handle multipoint geometry - points = [Coordinate(point[0], point[1]) for point in geometry['points']] - shape = parser(points, properties=properties) - - elif geometry_type_str == 'POLYLINE': - # Extract parts of the polyline and convert to list of Coordinates - lines = [] - for part in geometry['paths']: - line = [Coordinate(point[0], point[1]) for point in part] - lines.append(line) - - # Create the GeoLineString using the entire line - if len(lines) == 1: - shape = parser(lines[0], properties=properties) - else: - shape = MultiGeoLineString(lines, properties=properties) - - elif geometry_type_str == 'POLYGON': - # Handle MultiPolygon and single Polygon geometries - rings = [] - for ring in geometry['rings']: - # Convert each point in the ring to a Coordinate object - outline = [Coordinate(point[0], point[1]) for point in ring] - # Ensure the outline is closed (first point equals the last point) - if outline[0] != outline[-1]: - outline.append(outline[0]) - rings.append(outline) - - if len(rings) == 1: - # Create a GeoPolygon if there's only one part - outline = rings[0] - holes = rings[1:] if len(rings) > 1 else None - shape = parser(outline, holes=holes, properties=properties) - else: - # Create a MultiGeoPolygon if there are multiple parts - shape = MultiGeoPolygon([parser(ring) for ring in rings], properties=properties) - - else: - raise TypeError(f'Parser for {geometry_type_str} is not available at this time.') - - # Assign the 'dt' attribute if time fields are provided - start_time = getattr(row, time_start_property, None) if isinstance(time_start_property, str) else None - end_time = getattr(row, time_end_property, None) if isinstance(time_end_property, str) else None - shape.dt = _get_datetime_pandas(start_time, end_time) - shape._properties.update(_props) - _shapes.append(shape) + geometry = row.SHAPE + geometry_type_str = type(geometry).__name__.upper() + + if geometry_type_str not in _PARSER_MAP: + raise ValueError(f'Unsupported geometry type: {geometry_type_str}.') + + parser = _PARSER_MAP[geometry_type_str] + properties = {col: getattr(row, col) for col in columns} + time_start_value, time_end_value = None, None + if time_start_property is not None: + time_start_value = getattr(row, time_start_property, None) + + if time_end_property is not None: + time_end_value = getattr(row, time_end_property, None) - return _shapes + dt = None + if time_start_value and isinstance(time_start_value, str): + dt = TimeInterval.from_str(time_start_value, time_end_value, time_fmt) + + elif time_start_value: + dt = TimeInterval(time_start_value, time_end_value) + + return parser.from_featureclass( + geometry, + dt=dt, + properties=properties + ) def parse_arcpy_featureclass( - cursor, - fields: Optional[List[str]] = None, - time_start_property: Optional[str] = None, - time_end_property: Optional[str] = None, - _shapes: Optional[List[GeoShape]] = None, - _props: Optional[Dict[str, str]] = None, + row, + fields, + time_start_property: Optional[Union[str, datetime]] = None, + time_end_property: Optional[Union[str, datetime]] = None, + time_fmt: Optional[Union[str, List[str]]] = None, ): """ - Parses feature class data using arcpy and converts it into geostructures. + Parses an ArcGIS feature class row into a geospatial structure. Args: - cursor (arcpy.da.SearchCursor): - An arcpy.da.SearchCursor containing the feature class data. + row: The row from an ArcGIS feature class, typically obtained using an arcpy cursor. + fields: List of column names to extract attribute values from the row. + time_start_property: Optional; the column name or datetime representing the start time. + time_end_property: Optional; the column name or datetime representing the end time. + time_fmt: Optional; the format or list of formats for parsing time strings. - fields (List[str]): - List of field names to retrieve from the cursor. + Returns: + A geospatial object parsed from the feature class row, with attributes and time interval. - time_start_property (str, optional): - The name of the field containing the start time data. + Raises: + ValueError: If the geometry type is not supported. + """ + geometry_index = fields.index('SHAPE@') + time_start_index, time_end_index = None, None + + if time_start_property in fields: + time_start_index = fields.index(time_start_property) - time_end_property (str, optional): - The name of the field containing the end time data. + if time_end_property in fields: + time_end_index = fields.index(time_end_property) - _shapes (List[GeoShape], optional): - Internal use only. Mutated with geostructures as they're extracted from the feature class. + geometry = row.SHAPE + geometry_type_str = type(geometry).__name__.upper() - _props (Dict[str, str], optional): - Internal use only. Information about higher-level containers - to store as properties on the shape for traceability. + if geometry_type_str not in _PARSER_MAP: + raise ValueError(f'Unsupported geometry type: {geometry_type_str}.') - Returns: - List[GeoShape]: A list of GeoShape objects parsed from the feature class. - """ - if _shapes is None: - _shapes = [] - if _props is None: - _props = {} + parser = _PARSER_MAP[geometry_type_str] - geometry_index = fields.index('SHAPE@') - time_start_index = fields.index(time_start_property) if time_start_property in fields else None - time_end_index = fields.index(time_end_property) if time_end_property in fields else None - cursor.reset() - - for row in cursor: - geometry = row[geometry_index] - geometry_type_str = type(geometry).__name__.upper() - - if geometry_type_str not in _PARSER_MAP: - raise ValueError(f"Unsupported geometry type: {geometry_type_str}") - - parser = _PARSER_MAP[geometry_type_str] - properties = dict(zip(fields, row)) - del properties['SHAPE@'] - - if geometry_type_str == 'POINT': - lon, lat = geometry.X, geometry.Y - shape = parser(Coordinate(lon, lat), properties=properties) - - elif geometry_type_str == 'MULTIPOINT': - # Handle multipoint geometry - points = [Coordinate(point.X, point.Y) for point in geometry] - shape = parser(points, properties=properties) - - elif geometry_type_str == 'POLYLINE': - # Parse the polyline geometry from the ESRI format to GeoLineString - lines = [] - for part in geometry.getPart(): - # Convert each point in the part to a Coordinate object - line = [Coordinate(point.X, point.Y) for point in part] - lines.append(line) - - # Create the GeoLineString using the entire line - if len(lines) == 1: - shape = parser(lines[0], properties=properties) - else: - shape = MultiGeoLineString(lines, properties=properties) - - elif geometry_type_str == 'POLYGON': - # Parse the polygon geometry from the ESRI format to GeoPolygon or MultiGeoPolygon - rings = [] - for part in geometry.getPart(): - # Convert each point in the ring to a Coordinate object - outline = [Coordinate(point.X, point.Y) for point in part] - # Ensure the outline is closed (first point equals the last point) - if outline[0] != outline[-1]: - outline.append(outline[0]) - rings.append(outline) - - if len(rings) == 1: - # Create a GeoPolygon if there's only one part - outline = rings[0] - holes = rings[1:] if len(rings) > 1 else None - shape = parser(outline, holes=holes, properties=properties) - else: - # Create a MultiGeoPolygon if there are multiple parts - shape = MultiGeoPolygon([parser(ring) for ring in rings], properties=properties) - - else: - raise TypeError(f'Parser for {geometry_type_str} not available at this time.') - - # Assign the 'dt' attribute if time_start_property or time_end_property is provided and exists in the row - start_time = row[time_start_index] if isinstance(time_start_property, str) and time_start_index is not None else None - end_time = row[time_end_index] if isinstance(time_end_property, str) and time_end_index is not None else None - shape.dt = _get_datetime_pandas(start_time, end_time) - shape._properties.update(_props) - _shapes.append(shape) + properties = dict(zip(columns, row)) + del properties['SHAPE@'] + time_start_value, time_end_value = None, None + + if time_start_value is not None: + time_start_value = getattr(row, time_start_property, None) + + if time_end_value is not None: + time_end_value = getattr(row, time_end_property, None) - return _shapes + dt = None + if time_start_value and isinstance(time_start_value, str): + dt = TimeInterval.from_str(time_start_value, time_end_value, time_fmt) + + elif time_start_value: + dt = TimeInterval(time_start_value, time_end_value) + + return parser.from_featureclass( + geometry, + dt=dt, + properties=properties + ) def parse_fastkml( From 90b8c5db6db018f87c1c5d225b38b84dadc3841f Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:39:09 -0400 Subject: [PATCH 15/32] Update parsers.py --- geostructures/parsers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geostructures/parsers.py b/geostructures/parsers.py index 6496fe5..6509538 100644 --- a/geostructures/parsers.py +++ b/geostructures/parsers.py @@ -56,7 +56,7 @@ def parse_arcgis_featureclass( time_start_property: Optional[Union[str, datetime]] = None, time_end_property: Optional[Union[str, datetime]] = None, time_fmt: Optional[Union[str, List[str]]] = None, -): -> GeoShape +) -> GeoShape: """ Parses an ArcGIS feature class row into a geospatial structure. @@ -117,7 +117,7 @@ def parse_arcpy_featureclass( time_start_property: Optional[Union[str, datetime]] = None, time_end_property: Optional[Union[str, datetime]] = None, time_fmt: Optional[Union[str, List[str]]] = None, -): +) -> GeoShape: """ Parses an ArcGIS feature class row into a geospatial structure. From f2fae540a0e28765c358159a0df2a2f43a7f32af Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:28:31 -0400 Subject: [PATCH 16/32] Update structures.py from_featureclass added to GeoLineString, GeoPoint, and GeoPolygon --- geostructures/structures.py | 179 ++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/geostructures/structures.py b/geostructures/structures.py index d52a42d..8a5d87a 100644 --- a/geostructures/structures.py +++ b/geostructures/structures.py @@ -439,6 +439,103 @@ def copy(self): properties=copy.deepcopy(self._properties) ) + @classmethod + def from_featureclass( + cls, + geometry, + dt: Optional[GEOTIME_TYPE] = None, + properties: Optional[dict] = None, + ) -> Union['GeoPolygon', 'MultiGeoPolygon']: + """ + Creates a GeoPolygon or MultiGeoPolygon from ESRI formatted geometry + + Args: + geometry: + The geometry of the feature + + dt: (Optional) + TimeInterval from the parser + + properties: (Optional) + the columns and values of the attributes of the feature + """ + from geostructures._geometry import is_counter_clockwise + from geostructures.multistructures import MultiGeoPolygon + + def _get_rings_from_part(part): + idx, rings = 0, [] + while idx < len(part): + ring = [] + while idx < len(part) and part[idx] is not None: + ring.append(part[idx]) + idx += 1 + + rings.append(ring) + idx +=1 + + return rings + + shapes = [] + if isinstance(geometry, dict) and 'rings' in geometry: + holes, idx, outline = [], 0, None + while idx < len(geometry['rings']): + ring = Coordinate(*x) for x in geometry['rings'][idx]] + + if is_counter_clockwise(ring): + holes.append(GeoPolygon(ring)) + idx += 1 + continue + + if outline is not None: + shapes.append(GeoPolygon(outline, holes=holes or None)) + holes = [] + + outline = ring + idx += 1 + + if (not shapes) or outline != shapes[-1].bounds: + shapes.append(GeoPolygon(outline, holes=holes or None)) + + elif hasattr(geometry[0][0], 'X') and hasattr(geometry[0][0], 'Y'): + for part in geometry: + rings = _get_rings_from_part(part) + outline = [Coordinate(point.X, point.Y) for point in rings[0]] + holes = None + if len(rings) > 1: + holes = [ + GeoPolygon([ + Coordinate(point.X, point.Y) for point in ring + ]) for ring in rings[1:] + ] + + shapes.append(GeoPolygon(outline, holes=holes)) + + elif hasattr(geometry[0][0], 'centroid'):for part in geometry: + for part in geometry: + rings = _get_rings_from_part(part) + outline = [Coordinate(point.centroid.X, point.centroid.Y) for point in rings[0]] + holes = None + if len(rings) > 1: + holes = [ + GeoPolygon([ + Coordinate(point.centroid.X, point.centroid.Y) for point in ring + ]) for ring in rings[1:] + ] + + shapes.append(GeoPolygon(outline, holes=holes)) + + else: + raise ValueError('Unable to extract shape from provided format.') + + if len(shapes) > 1: + return MultiGeoPolygon(shapes, dt=dt, properties=properties) + + shape = shapes[0] + shape._properties = properties + shape.dt = dt + + return shape + @classmethod def from_geojson( cls, @@ -1352,6 +1449,54 @@ def copy(self) -> 'GeoLineString': properties=copy.deepcopy(self._properties) ) + @classmethod + def from_featureclass( + cls, + geometry, + dt: Optional[GEOTIME_TYPE] = None, + properties: Optional[dict] = None, + ) -> Union['GeoLineString', 'MultiGeoLineString']: + """ + Creates a GeoLineString or MultiGeoLineString from ESRI formatted geometry + + Args: + geometry: + The geometry of the feature + + dt: (Optional) + TimeInterval from the parser + + properties: (Optional) + the columns and values of the attributes of the feature + """ + lines = [] + if isinstance(geometry, dict) and 'paths' in geometry: + for part in geometry['paths']: + line = [Coordinate(point[0], point[1]) for point in part] + lines.append(GeoLineString(line)) + + elif hasattr(geometry[0][0], 'X') and hasattr(geometry[0][0], 'Y'): + for part in geometry: + line = [Coordinate(point.X, point.Y) for point in part] + lines.append(GeoLineString(line)) + + elif hasattr(geometry[0][0], 'centroid'): + for part in geometry: + line = [Coordinate(point.centroid.X, point.centroid.Y) for point in part] + lines.append(GeoLineString(line)) + + else: + raise ValueError('Unable to extract shape from provided format.') + + if len(lines) > 1: + return MultiGeoLineString(lines, dt=dt, properties=properties) + + line = lines[0] + line._properties = properties + line.dt = dt + + return lines + @classmethod def from_geojson( cls, @@ -1596,6 +1741,40 @@ def intersects_shape(self, shape: 'GeoShape', **kwargs) -> bool: return self == shape return self in shape + @classmethod + def from_featureclass( + cls, + geometry, + dt: Optional[GEOTIME_TYPE] = None, + properties: Optional[dict] = None, + ) -> 'GeoPoint': + """ + Creates a GeoPoint from ESRI formatted geometry + + Args: + geometry: + The geometry of the feature + + dt: (Optional) + TimeInterval from the parser + + properties: (Optional) + the columns and values of the attributes of the feature + """ + if isinstance(geometry, dict) and 'x' in geometry and 'y' in geometry: + coord = Coordinate(geometry['x'], geometry['y']) + + elif hasattr(geometry, 'X') and hasattr(geometry, 'Y'): + coord = Coordinate(geometry.X, geometry.Y) + + elif hasattr(geometry, 'centroid'): + coord = Coordinate(geometry.centroid.X, geometry.centroid.Y) + + else: + raise ValueError('Unable to extract shape from provided format.') + + return GeoPoint(coord, dt=dt, properties=properties) + @classmethod def from_geojson( cls, From f1200608a7f90f0744673df9e1eb5d41222beac1 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:17:55 -0400 Subject: [PATCH 17/32] Update collections.py from_arcgis_featureclass and from_arcpy_featureclass methods --- geostructures/collections.py | 146 +++++++++++++++++++++++++++++------ 1 file changed, 122 insertions(+), 24 deletions(-) diff --git a/geostructures/collections.py b/geostructures/collections.py index f94e333..e09c9a5 100644 --- a/geostructures/collections.py +++ b/geostructures/collections.py @@ -141,7 +141,8 @@ def from_arcgis_featureclass( cls, feature_class_path: str, time_start_property: Optional[str] = None, - time_end_property: Optional[str] = None + time_end_property: Optional[str] = None, + time_fmt: Optional[Union[str, List[str]]] = None, ): """ Creates an instance of the class from an ArcGIS feature class. @@ -153,6 +154,8 @@ def from_arcgis_featureclass( The name of the field containing the start time data. Defaults to None. time_end_property (Optional[str]): The name of the field containing the end time data. Defaults to None. + time_fmt (Optional[str] or [List[str]]): + The string format(s) of the time properties if they are strs. Defaults to None. Returns: An instance of the class populated with geoshapes parsed from the feature class. @@ -161,21 +164,66 @@ def from_arcgis_featureclass( import pandas as pd from geostructures.parsers import parse_arcgis_featureclass + _shapes = [] # Convert the feature class into a Spatially Enabled DataFrame (SEDF) for further processing sedf = pd.DataFrame.spatial.from_featureclass(feature_class_path) + property_columns = [col for col in sedf.columns if col != 'SHAPE'] + + # Get time values to determine the fmt needed if they are not strings and fmt was not provided + time_start_value, time_end_value = None, None + if time_start_property is not None: + time_start_value = getattr(sedf.iloc[0], time_start_property, None) + + if time_end_property is not None: + time_end_value = getattr(sedf.iloc[0], time_start_property, None) + + # Pull time format for time_start_property, if it is a string. + if time_start_value and isinstance(time_start_value, str): + if time_fmt is None: + time_fmt = [TimeInterval._get_timeformat(time_start_value)] + + if time_end_value and not isinstance(time_end_value, str): + raise TypeError( + f'Time formats cannot be mixed, ' + f'Start Time: {type(time_start_value)}, ' + f'End Time: {type(time_end_value)}' + ) - # Parse the SEDF to extract geoshapes and convert them into the desired format - geoshapes = parse_arcgis_featureclass(sedf, time_start_property, time_end_property) + # Pull time format for time_end_property, if it is a string. + if time_end_value and isinstance(time_end_value, str): + end_time_fmt = [TimeInterval._get_timeformat(time_end_value)] + if time_fmt != end_time_fmt: + if end_time_fmt not in time_fmt: + time_fmt.append(fmt for fmt in end_time_fmt) + + if time_start_value and not is instance(time_start_value, str): + raise TypeError( + f'Time formats cannot be mixed, ' + f'Start Time: {type(time_start_value)}, ' + f'End Time: {type(time_end_value)}' + ) - # Return an instance of the class with the parsed geoshapes - return cls(geoshapes) + # Parse each row in SEDF to extract geoshapes and convert them into GeoShapes + for row in sedf.itertuples(): + _shapes.append( + parse_arcgis_featureclass( + row, + property_columns, + time_start_property, + time_end_property, + time_fmt + ) + ) + + return cls(_shapes) @classmethod def from_arcpy_featureclass( cls, feature_class_path: str, time_start_property: Optional[str] = None, - time_end_property: Optional[str] = None + time_end_property: Optional[str] = None, + time_fmt: Optional[Union[str, List[str]]] = None, ): """ Creates an instance of the class from an ArcGIS feature class. @@ -187,33 +235,83 @@ def from_arcpy_featureclass( The name of the field containing the start time data. Defaults to None. time_end_property (Optional[str]): The name of the field containing the end time data. Defaults to None. + time_fmt (Optional[str] or [List[str]]): + The string format(s) of the time properties if they are strs. Defaults to None. Returns: An instance of the class populated with geoshapes parsed from the feature class. """ - import arcpy + from arcpy # noqa: F401 from geostructures.parsers import parse_arcpy_featureclass - # Create a set for time properties to ensure uniqueness - time_properties = set() - if time_start_property: - time_properties.add(time_start_property) - if time_end_property: - time_properties.add(time_end_property) - - # Initialize fields list with 'SHAPE@' and add unique time properties - fields = ['SHAPE@'] + list(time_properties) - - # Add all other fields from the feature class, ensuring no duplicates + _shapes = [] + # Convert the feature class into a Spatially Enabled DataFrame (SEDF) for further processing + fields = ['SHAPE@'] fields.extend([f.name for f in arcpy.ListFields(feature_class_path) if f.name not in fields]) + cursor = arcpy.da.SearchCursor(feature_class_path, fields) + if time_start_property and time_start_property not in fields: + raise ValueError(f'Invalid start time provided: {time_start_property}.') + + if time_end_property and time_end_property not in fields: + raise ValueError(f'Invalid end time provided: {time_end_property}.') + + # Get time values to determine the fmt needed if they are not strings and fmt was not provided + for row in cursor: + time_start_index, time_end_index = None, None + if time_start_property is not None: + time_start_index = fields.index(time_start_property) + + if time_end_property is not None: + time_end_index = fields.index(time_end_property) + + time_start_value, time_end_value = None, None + if time_start_index is not None: + time_start_value = row[time_start_index] + + if time_end_index is not None: + time_end_value = row[time_end_value] + + # Pull time format for time_start_property, if it is a string. + if time_start_value and isinstance(time_start_value, str): + if time_fmt is None: + time_fmt = [TimeInterval._get_timeformat(time_start_value)] + + if time_end_value and not isinstance(time_end_value, str): + raise TypeError( + f'Time formats cannot be mixed, ' + f'Start Time: {type(time_start_value)}, ' + f'End Time: {type(time_end_value)}' + ) + + # Pull time format for time_end_property, if it is a string. + if time_end_value and isinstance(time_end_value, str): + end_time_fmt = [TimeInterval._get_timeformat(time_end_value)] + if time_fmt != end_time_fmt: + if end_time_fmt not in time_fmt: + time_fmt.append(fmt for fmt in end_time_fmt) + + if time_start_value and not is instance(time_start_value, str): + raise TypeError( + f'Time formats cannot be mixed, ' + f'Start Time: {type(time_start_value)}, ' + f'End Time: {type(time_end_value)}' + ) - # Use an arcpy SearchCursor to iterate over the feature class rows and extract the relevant fields - with arcpy.da.SearchCursor(feature_class_path, fields) as cursor: - # Parse the feature class using the cursor and provided fields - geoshapes = parser_arcpy_featureclass(cursor, fields, time_start_property, time_end_property) + break + cursor.reset() + # Parse each row in cursor to extract geoshapes + for row in cursor: + _shapes.append( + parse_arcpy_featureclass( + row, + fields, + time_start_property, + time_end_property, + time_fmt + ) + ) - # Return an instance of the class with the parsed geoshapes - return cls(geoshapes) + return cls(_shapes) @classmethod def from_geojson( From fa27e38b3247883bf3a110255ef3dff2f34dc6f5 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:26:10 -0400 Subject: [PATCH 18/32] Update parsers.py --- geostructures/parsers.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/geostructures/parsers.py b/geostructures/parsers.py index 6509538..723aec8 100644 --- a/geostructures/parsers.py +++ b/geostructures/parsers.py @@ -12,6 +12,7 @@ from geostructures.coordinates import Coordinate from geostructures.structures import GeoPolygon, GeoPoint, GeoLineString from geostructures.multistructures import MultiGeoPoint, MultiGeoPolygon, MultiGeoLineString +from geostructures.time import TimeInterval from geostructures.typing import GeoShape, SimpleShape @@ -53,8 +54,8 @@ def _get_datetime_pandas(start_time, end_time): def parse_arcgis_featureclass( row, columns, - time_start_property: Optional[Union[str, datetime]] = None, - time_end_property: Optional[Union[str, datetime]] = None, + time_start_property: Optional[Union[str, 'datetime']] = None, + time_end_property: Optional[Union[str, 'datetime']] = None, time_fmt: Optional[Union[str, List[str]]] = None, ) -> GeoShape: """ @@ -82,6 +83,8 @@ def parse_arcgis_featureclass( Raises: ValueError: If the geometry type is not supported. """ + from datetime import datetime + geometry = row.SHAPE geometry_type_str = type(geometry).__name__.upper() @@ -114,8 +117,8 @@ def parse_arcgis_featureclass( def parse_arcpy_featureclass( row, fields, - time_start_property: Optional[Union[str, datetime]] = None, - time_end_property: Optional[Union[str, datetime]] = None, + time_start_property: Optional[Union[str, 'datetime']] = None, + time_end_property: Optional[Union[str, 'datetime']] = None, time_fmt: Optional[Union[str, List[str]]] = None, ) -> GeoShape: """ @@ -134,6 +137,8 @@ def parse_arcpy_featureclass( Raises: ValueError: If the geometry type is not supported. """ + from datetime import datetime + geometry_index = fields.index('SHAPE@') time_start_index, time_end_index = None, None @@ -143,18 +148,16 @@ def parse_arcpy_featureclass( if time_end_property in fields: time_end_index = fields.index(time_end_property) - geometry = row.SHAPE + geometry = row[geometry_index] geometry_type_str = type(geometry).__name__.upper() if geometry_type_str not in _PARSER_MAP: raise ValueError(f'Unsupported geometry type: {geometry_type_str}.') parser = _PARSER_MAP[geometry_type_str] - - properties = dict(zip(columns, row)) + properties = dict(zip(fields, row)) del properties['SHAPE@'] time_start_value, time_end_value = None, None - if time_start_value is not None: time_start_value = getattr(row, time_start_property, None) From 00a1c16a97c649c47747ce9de96850d40e9ca02f Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:29:28 -0400 Subject: [PATCH 19/32] Update parsers.py --- geostructures/parsers.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/geostructures/parsers.py b/geostructures/parsers.py index 723aec8..de1c10b 100644 --- a/geostructures/parsers.py +++ b/geostructures/parsers.py @@ -6,10 +6,10 @@ import json import re +from datetime import datetime from typing import cast, Any, Dict, List, Optional, Union from geostructures.collections import FeatureCollection -from geostructures.coordinates import Coordinate from geostructures.structures import GeoPolygon, GeoPoint, GeoLineString from geostructures.multistructures import MultiGeoPoint, MultiGeoPolygon, MultiGeoLineString from geostructures.time import TimeInterval @@ -82,9 +82,7 @@ def parse_arcgis_featureclass( Raises: ValueError: If the geometry type is not supported. - """ - from datetime import datetime - + """ geometry = row.SHAPE geometry_type_str = type(geometry).__name__.upper() @@ -137,8 +135,6 @@ def parse_arcpy_featureclass( Raises: ValueError: If the geometry type is not supported. """ - from datetime import datetime - geometry_index = fields.index('SHAPE@') time_start_index, time_end_index = None, None From 59ec916783644dc033926ca2935a1f9cc246d633 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:31:03 -0400 Subject: [PATCH 20/32] Update collections.py --- geostructures/collections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geostructures/collections.py b/geostructures/collections.py index e09c9a5..8454a12 100644 --- a/geostructures/collections.py +++ b/geostructures/collections.py @@ -196,7 +196,7 @@ def from_arcgis_featureclass( if end_time_fmt not in time_fmt: time_fmt.append(fmt for fmt in end_time_fmt) - if time_start_value and not is instance(time_start_value, str): + if time_start_value and not isinstance(time_start_value, str): raise TypeError( f'Time formats cannot be mixed, ' f'Start Time: {type(time_start_value)}, ' From e5a2fce85e8fd95ca5ee6b7648b709283af95e1f Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:32:38 -0400 Subject: [PATCH 21/32] Update parsers.py --- geostructures/parsers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/geostructures/parsers.py b/geostructures/parsers.py index de1c10b..c90c753 100644 --- a/geostructures/parsers.py +++ b/geostructures/parsers.py @@ -82,7 +82,7 @@ def parse_arcgis_featureclass( Raises: ValueError: If the geometry type is not supported. - """ + """ geometry = row.SHAPE geometry_type_str = type(geometry).__name__.upper() @@ -137,7 +137,7 @@ def parse_arcpy_featureclass( """ geometry_index = fields.index('SHAPE@') time_start_index, time_end_index = None, None - + if time_start_property in fields: time_start_index = fields.index(time_start_property) @@ -155,10 +155,10 @@ def parse_arcpy_featureclass( del properties['SHAPE@'] time_start_value, time_end_value = None, None if time_start_value is not None: - time_start_value = getattr(row, time_start_property, None) + time_start_value = row[time_start_index] if time_end_value is not None: - time_end_value = getattr(row, time_end_property, None) + time_end_value = row[time_end_index] dt = None if time_start_value and isinstance(time_start_value, str): From c3b55b196bde84b019a69fa53f4968e7b059662a Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:35:01 -0400 Subject: [PATCH 22/32] Update collections.py --- geostructures/collections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geostructures/collections.py b/geostructures/collections.py index 8454a12..e9dbf23 100644 --- a/geostructures/collections.py +++ b/geostructures/collections.py @@ -241,7 +241,7 @@ def from_arcpy_featureclass( Returns: An instance of the class populated with geoshapes parsed from the feature class. """ - from arcpy # noqa: F401 + import arcpy # noqa: F401 from geostructures.parsers import parse_arcpy_featureclass _shapes = [] From 79adb37e0aa829835d80c8149244f15ec2b5bc9e Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:36:13 -0400 Subject: [PATCH 23/32] Update structures.py --- geostructures/structures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geostructures/structures.py b/geostructures/structures.py index 8a5d87a..cf889b2 100644 --- a/geostructures/structures.py +++ b/geostructures/structures.py @@ -479,7 +479,7 @@ def _get_rings_from_part(part): if isinstance(geometry, dict) and 'rings' in geometry: holes, idx, outline = [], 0, None while idx < len(geometry['rings']): - ring = Coordinate(*x) for x in geometry['rings'][idx]] + ring = [Coordinate(*x) for x in geometry['rings'][idx]] if is_counter_clockwise(ring): holes.append(GeoPolygon(ring)) From c54e40199623c62fc37401b37f4ffdd597679914 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:37:19 -0400 Subject: [PATCH 24/32] Update structures.py --- geostructures/structures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geostructures/structures.py b/geostructures/structures.py index cf889b2..b51d8bb 100644 --- a/geostructures/structures.py +++ b/geostructures/structures.py @@ -510,7 +510,7 @@ def _get_rings_from_part(part): shapes.append(GeoPolygon(outline, holes=holes)) - elif hasattr(geometry[0][0], 'centroid'):for part in geometry: + elif hasattr(geometry[0][0], 'centroid'): for part in geometry: rings = _get_rings_from_part(part) outline = [Coordinate(point.centroid.X, point.centroid.Y) for point in rings[0]] From d16fc8b35a264e13772206ebc8931d0d16cf4a2d Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:37:50 -0400 Subject: [PATCH 25/32] Update collections.py --- geostructures/collections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geostructures/collections.py b/geostructures/collections.py index e9dbf23..49f8576 100644 --- a/geostructures/collections.py +++ b/geostructures/collections.py @@ -290,7 +290,7 @@ def from_arcpy_featureclass( if end_time_fmt not in time_fmt: time_fmt.append(fmt for fmt in end_time_fmt) - if time_start_value and not is instance(time_start_value, str): + if time_start_value and not isinstance(time_start_value, str): raise TypeError( f'Time formats cannot be mixed, ' f'Start Time: {type(time_start_value)}, ' From d7814fe8b81cb93ba08adadd7a906fafec7569c4 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:39:47 -0400 Subject: [PATCH 26/32] Update collections.py --- geostructures/collections.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/geostructures/collections.py b/geostructures/collections.py index 49f8576..9b7f462 100644 --- a/geostructures/collections.py +++ b/geostructures/collections.py @@ -263,19 +263,19 @@ def from_arcpy_featureclass( if time_end_property is not None: time_end_index = fields.index(time_end_property) - + time_start_value, time_end_value = None, None if time_start_index is not None: time_start_value = row[time_start_index] if time_end_index is not None: time_end_value = row[time_end_value] - + # Pull time format for time_start_property, if it is a string. if time_start_value and isinstance(time_start_value, str): if time_fmt is None: time_fmt = [TimeInterval._get_timeformat(time_start_value)] - + if time_end_value and not isinstance(time_end_value, str): raise TypeError( f'Time formats cannot be mixed, ' @@ -289,7 +289,7 @@ def from_arcpy_featureclass( if time_fmt != end_time_fmt: if end_time_fmt not in time_fmt: time_fmt.append(fmt for fmt in end_time_fmt) - + if time_start_value and not isinstance(time_start_value, str): raise TypeError( f'Time formats cannot be mixed, ' @@ -297,7 +297,8 @@ def from_arcpy_featureclass( f'End Time: {type(time_end_value)}' ) - break + break + cursor.reset() # Parse each row in cursor to extract geoshapes for row in cursor: From c90fdbc7a12ac4d23717e518d917357763554377 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:40:50 -0400 Subject: [PATCH 27/32] Update structures.py --- geostructures/structures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geostructures/structures.py b/geostructures/structures.py index b51d8bb..bee2728 100644 --- a/geostructures/structures.py +++ b/geostructures/structures.py @@ -35,6 +35,7 @@ circumscribing_circle_for_polygon, do_edges_intersect, find_line_intersection, is_counter_clockwise ) +from geostructures.multistructures import MultiGeoPolygon, MultiGeoLineString from geostructures.utils.functions import round_half_up, get_dt_from_geojson_props, is_sub_list from geostructures.utils.logging import warn_once @@ -460,7 +461,6 @@ def from_featureclass( the columns and values of the attributes of the feature """ from geostructures._geometry import is_counter_clockwise - from geostructures.multistructures import MultiGeoPolygon def _get_rings_from_part(part): idx, rings = 0, [] From 5de49ac6ca76224d89075099ce511ddae1d314a7 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:42:45 -0400 Subject: [PATCH 28/32] Update structures.py --- geostructures/structures.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/geostructures/structures.py b/geostructures/structures.py index bee2728..eba1929 100644 --- a/geostructures/structures.py +++ b/geostructures/structures.py @@ -13,7 +13,7 @@ from functools import cached_property import math import statistics -from typing import cast, Any, Dict, List, Optional, Tuple, Sequence, TYPE_CHECKING +from typing import cast, Any, Dict, List, Optional, Tuple, Sequence, TYPE_CHECKING, Union import numpy as np @@ -471,10 +471,10 @@ def _get_rings_from_part(part): idx += 1 rings.append(ring) - idx +=1 + idx += 1 return rings - + shapes = [] if isinstance(geometry, dict) and 'rings' in geometry: holes, idx, outline = [], 0, None @@ -507,7 +507,7 @@ def _get_rings_from_part(part): Coordinate(point.X, point.Y) for point in ring ]) for ring in rings[1:] ] - + shapes.append(GeoPolygon(outline, holes=holes)) elif hasattr(geometry[0][0], 'centroid'): From f525470f5d979a17bf60b242634aaf0633cfae77 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:43:50 -0400 Subject: [PATCH 29/32] Update structures.py --- geostructures/structures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geostructures/structures.py b/geostructures/structures.py index eba1929..6732319 100644 --- a/geostructures/structures.py +++ b/geostructures/structures.py @@ -521,7 +521,7 @@ def _get_rings_from_part(part): Coordinate(point.centroid.X, point.centroid.Y) for point in ring ]) for ring in rings[1:] ] - + shapes.append(GeoPolygon(outline, holes=holes)) else: From e96e3ebdb83032aa4ed9d40800d5531e4517ac34 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:49:37 -0400 Subject: [PATCH 30/32] Update structures.py --- geostructures/structures.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/geostructures/structures.py b/geostructures/structures.py index 6732319..43d8bfc 100644 --- a/geostructures/structures.py +++ b/geostructures/structures.py @@ -35,7 +35,6 @@ circumscribing_circle_for_polygon, do_edges_intersect, find_line_intersection, is_counter_clockwise ) -from geostructures.multistructures import MultiGeoPolygon, MultiGeoLineString from geostructures.utils.functions import round_half_up, get_dt_from_geojson_props, is_sub_list from geostructures.utils.logging import warn_once @@ -446,7 +445,7 @@ def from_featureclass( geometry, dt: Optional[GEOTIME_TYPE] = None, properties: Optional[dict] = None, - ) -> Union['GeoPolygon', 'MultiGeoPolygon']: + ) -> 'GeoShape': """ Creates a GeoPolygon or MultiGeoPolygon from ESRI formatted geometry @@ -461,7 +460,7 @@ def from_featureclass( the columns and values of the attributes of the feature """ from geostructures._geometry import is_counter_clockwise - + from geostructures.multistructures import MultiGeoPolygon def _get_rings_from_part(part): idx, rings = 0, [] while idx < len(part): @@ -1455,7 +1454,7 @@ def from_featureclass( geometry, dt: Optional[GEOTIME_TYPE] = None, properties: Optional[dict] = None, - ) -> Union['GeoLineString', 'MultiGeoLineString']: + ) -> 'GeoShape': """ Creates a GeoLineString or MultiGeoLineString from ESRI formatted geometry @@ -1469,6 +1468,8 @@ def from_featureclass( properties: (Optional) the columns and values of the attributes of the feature """ + from geostructures.multistructures import MultiGeoLineString + lines = [] if isinstance(geometry, dict) and 'paths' in geometry: for part in geometry['paths']: From 87a697c75e16b179edf24aa7095b8060490e36f9 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:50:42 -0400 Subject: [PATCH 31/32] Update structures.py --- geostructures/structures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/geostructures/structures.py b/geostructures/structures.py index 43d8bfc..c813e4a 100644 --- a/geostructures/structures.py +++ b/geostructures/structures.py @@ -461,6 +461,7 @@ def from_featureclass( """ from geostructures._geometry import is_counter_clockwise from geostructures.multistructures import MultiGeoPolygon + def _get_rings_from_part(part): idx, rings = 0, [] while idx < len(part): From 944656f3dd563d3940140f9d7d1e92702c2e1dc8 Mon Sep 17 00:00:00 2001 From: Rkleisley <63410265+Rkleisley@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:51:34 -0400 Subject: [PATCH 32/32] Update structures.py --- geostructures/structures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geostructures/structures.py b/geostructures/structures.py index c813e4a..db188da 100644 --- a/geostructures/structures.py +++ b/geostructures/structures.py @@ -13,7 +13,7 @@ from functools import cached_property import math import statistics -from typing import cast, Any, Dict, List, Optional, Tuple, Sequence, TYPE_CHECKING, Union +from typing import cast, Any, Dict, List, Optional, Tuple, Sequence, TYPE_CHECKING import numpy as np