From 0d9f0ede66578f80fffb8e53fdf087d4b28c4850 Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Thu, 30 Oct 2025 11:49:44 -0400 Subject: [PATCH 1/3] fix output_conf/conf_extra resolutions + fix chaining of DGGRS params across DGGRID calls + fix cells_for_geo_points attributes override (fixes #39, fixes #40) --- .gitignore | 1 + dggrid4py/dggrid_runner.py | 49 +++++++----- tests/test_dggrid.py | 157 ++++++++++++++++++++++++++++++++++++- 3 files changed, 187 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index be624e1..e9f446f 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ .spyderproject .spyproject .tox/ +.run/ .venv .vscode.ws.code-workspace .webassets-cache diff --git a/dggrid4py/dggrid_runner.py b/dggrid4py/dggrid_runner.py index c0584a5..5192090 100644 --- a/dggrid4py/dggrid_runner.py +++ b/dggrid4py/dggrid_runner.py @@ -987,19 +987,22 @@ def dgapi_grid_transform( subset_conf.update(input_extras) # transform output_types + output_conf = {} if ( - 'output_file_name' in output_conf.keys() and - 'output_address_type' in output_conf.keys() and - output_conf['output_address_type'] in self.output_address_types + 'output_file_name' in conf_extra.keys() and + 'output_address_type' in conf_extra.keys() and + conf_extra['output_address_type'] in self.output_address_types ): - for elem in filter(lambda x: x.startswith('output_') , output_conf.keys()): - metafile.append(f"{elem} " + str(output_conf[elem])) + for elem in filter(lambda x: x.startswith('output_') , conf_extra.keys()): + output_conf[elem] = conf_extra[elem] + metafile.append(f"{elem} " + str(conf_extra[elem])) else: raise ValueError('no output filename or type given') output_extras = self.check_output_extra_fields(conf_extra) if output_extras: for elem, value in output_extras.items(): + output_conf[elem] = value metafile.append(f"{elem} " + str(value)) result = self.run(metafile) @@ -1077,19 +1080,22 @@ def dgapi_point_value_binning( subset_conf.update(input_extras) # transform output_types + output_conf = {} if ( - 'output_file_name' in output_conf.keys() and - 'output_address_type' in output_conf.keys() and - output_conf['output_address_type'] in self.output_address_types + 'output_file_name' in conf_extra.keys() and + 'output_address_type' in conf_extra.keys() and + conf_extra['output_address_type'] in self.output_address_types ): - for elem in filter(lambda x: x.startswith('output_') , output_conf.keys()): - metafile.append(f"{elem} " + output_conf[elem]) + for elem in filter(lambda x: x.startswith('output_') , conf_extra.keys()): + output_conf[elem] = conf_extra[elem] + metafile.append(f"{elem} " + conf_extra[elem]) else: raise ValueError('no output filename or type given') output_extras = self.check_output_extra_fields(conf_extra) if output_extras: for elem, value in output_extras.items(): + output_conf[elem] = conf_extra[elem] metafile.append(f"{elem} " + str(value)) result = self.run(metafile) @@ -1144,19 +1150,22 @@ def dgapi_pres_binning( subset_conf.update(input_extras) # transform output_types + output_conf = {} if ( - 'output_file_name' in output_conf.keys() and - 'output_address_type' in output_conf.keys() and - output_conf['output_address_type'] in self.output_address_types + 'output_file_name' in conf_extra.keys() and + 'output_address_type' in conf_extra.keys() and + conf_extra['output_address_type'] in self.output_address_types ): - for elem in filter(lambda x: x.startswith('output_') , output_conf.keys()): - metafile.append(f"{elem} " + output_conf[elem]) + for elem in filter(lambda x: x.startswith('output_') , conf_extra.keys()): + output_conf[elem] = conf_extra[elem] + metafile.append(f"{elem} " + conf_extra[elem]) else: raise ValueError('no output filename or type given') output_extras = self.check_output_extra_fields(conf_extra) if output_extras: for elem, value in output_extras.items(): + output_conf[elem] = conf_extra[elem] metafile.append(f"{elem} " + str(value)) result = self.run(metafile) @@ -1848,7 +1857,7 @@ def cells_for_geo_points( dggs.update(**conf_extra, strict=True) cols = set(geodf_points_wgs84.columns.tolist()) - cols = cols - set('geometry') + cols = cols - {'geometry'} geodf_points_wgs84['lon'] = geodf_points_wgs84['geometry'].x geodf_points_wgs84['lat'] = geodf_points_wgs84['geometry'].y cols_ordered = ['lon', 'lat'] @@ -1905,15 +1914,19 @@ def cells_for_geo_points( return geodf_points_wgs84 else: # grid_gen from seqnums + dggs_conf = dggs.to_dict() gdf = self.grid_cell_polygons_from_cellids( cell_id_list=cell_id_list, - dggs_type=dggs_type, + # dggs_type=dggs_type, passed via dggs_conf resolution=resolution, mixed_aperture_level=mixed_aperture_level, input_address_type=output_address_type, - output_address_type=self.output_address_types, + output_address_type=output_address_type, + **dggs_conf, # ensure any extra parameters are passed on ) try: + # avoid conflict between input 'name' column and generated 'name' Zone ID + gdf.rename(columns={'name': 'zone'}, inplace=True) for col in cols_ordered: gdf[col] = geodf_points_wgs84[col].values except Exception: diff --git a/tests/test_dggrid.py b/tests/test_dggrid.py index 00806cf..b1d3fc5 100644 --- a/tests/test_dggrid.py +++ b/tests/test_dggrid.py @@ -1,13 +1,18 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- import decimal +import os + import pytest import shapely +import geopandas as gpd +from geopandas.testing import assert_geodataframe_equal from dggrid4py import DGGRIDv8, Dggs -from dggrid4py.dggrid_runner import output_hier_ndx_forms -dggrid = DGGRIDv8() +dggrid_path = os.getenv("DGGRID_PATH") +dggrid = DGGRIDv8(executable=dggrid_path) + def mock_dggrid_run(__metafile): return 0 @@ -264,3 +269,151 @@ def mock_dggrid_grid_gen_run(__metafile): # "output_hier_ndx_form DIGIT_STRING", "point_output_type NONE" } + + +def test_cells_for_geo_points(monkeypatch): + dgapi_grid_transform = dggrid.dgapi_grid_transform + dgapi_grid_gen = dggrid.dgapi_grid_gen + + dgapi_grid_transform_metafile = [] + dgapi_grid_transform_out_conf = {} + + dgapi_grid_gen_metafile = [] + dgapi_grid_gen_out_conf = {} + + def mock_dgapi_grid_transform(*args, **kwargs): + _res = dgapi_grid_transform(*args, **kwargs) + dgapi_grid_transform_metafile[:] = _res["metafile"] + dgapi_grid_transform_out_conf.update(_res["output_conf"]) + return _res + + def mock_dgapi_grid_gen(*args, **kwargs): + _res = dgapi_grid_gen(*args, **kwargs) + dgapi_grid_gen_metafile[:] = _res["metafile"] + dgapi_grid_gen_out_conf.update(_res["output_conf"]) + return _res + + monkeypatch.setattr(dggrid, "dgapi_grid_transform", mock_dgapi_grid_transform) + monkeypatch.setattr(dggrid, "dgapi_grid_gen", mock_dgapi_grid_gen) + + points = [shapely.Point(20.5, 57.5), shapely.Point(21.0, 58.0)] + geodf_points_wgs84 = gpd.GeoDataFrame({'name': ['A', 'B']}, geometry=points, crs='EPSG:4326') + + result = dggrid.cells_for_geo_points( + geodf_points_wgs84, + cell_ids_only=False, + dggs_type="ISEA7H", + resolution=5, + # use string to preserve precision and training zeros explicitly + dggs_vert0_azimuth=0.0, + dggs_vert0_lat="58.282525588538994675786", # default: 58.28252559 + dggs_vert0_lon="11.20", # default: 11.25 + ) + + # pre-check temp file paths to ignore in check of specific values + meta_args = dict([line.split(" ", 1) for line in dgapi_grid_transform_metafile]) + dgapi_grid_transform_input_file_name = meta_args.pop("input_file_name") + dgapi_grid_transform_output_file_name = meta_args.pop("output_file_name") + assert dgapi_grid_transform_input_file_name.startswith("/tmp/dggrid") + assert dgapi_grid_transform_output_file_name.startswith("/tmp/dggrid") + dgapi_grid_transform_metafile_patched = [f"{key} {val}" for key, val in meta_args.items()] + + meta_args = dict([line.split(" ", 1) for line in dgapi_grid_gen_metafile]) + dgapi_grid_gen_cell_output_file_name = meta_args.pop("cell_output_file_name") + dgapi_grid_gen_clip_region_files = meta_args.pop("clip_region_files") + assert dgapi_grid_gen_cell_output_file_name.startswith("/tmp/dggrid") + assert dgapi_grid_gen_clip_region_files.startswith("/tmp/dggrid") + dgapi_grid_gen_metafile_patched = [f"{key} {val}" for key, val in meta_args.items()] + + assert dgapi_grid_transform_output_file_name != dgapi_grid_gen_cell_output_file_name + + assert set(dgapi_grid_transform_metafile_patched) == { + "dggrid_operation TRANSFORM_POINTS", + "dggs_type ISEA7H", + "dggs_proj ISEA", + "dggs_aperture 7", + "dggs_topology HEXAGON", + "dggs_res_spec 5", + "precision 7", + # following set explicitly by input parameters + "dggs_orient_specify_type SPECIFIED", + "dggs_vert0_azimuth 0.0", + "dggs_vert0_lat 58.282525588538994675786", + "dggs_vert0_lon 11.20", + # following enforced by function to align with input GeoDataFrame + "input_address_type GEO", + "input_delimiter \" \"", + "output_address_type SEQNUM", + "output_delimiter \",\"", + # pre-checked temp file locations + # "input_file_name": "/tmp/dggrid/...", + # "output_file_name": "/tmp/dggrid/...", + } + + assert dgapi_grid_transform_out_conf == { + "output_address_type": "SEQNUM", + "output_delimiter": "\",\"", + "output_file_name": dgapi_grid_transform_output_file_name, + } + + assert set(dgapi_grid_gen_metafile_patched) == { + "dggrid_operation GENERATE_GRID", + "dggs_type ISEA7H", + "dggs_proj ISEA", + "dggs_aperture 7", + "dggs_topology HEXAGON", + "dggs_res_spec 5", + "precision 7", + # following set explicitly by input parameters + "dggs_orient_specify_type SPECIFIED", + "dggs_vert0_azimuth 0.0", + "dggs_vert0_lat 58.282525588538994675786", + "dggs_vert0_lon 11.20", + # following enforced by function to align with input of previous transform step + "clip_subset_type SEQNUMS", + "cell_output_type GDAL", + "cell_output_gdal_format FlatGeobuf", + "point_output_type NONE", + # pre-checked temp file locations + # "cell_output_file_name": "/tmp/dggrid/...", + } + + assert dgapi_grid_gen_out_conf == { + "cell_output_type": "GDAL", + "cell_output_gdal_format": "FlatGeobuf", + "cell_output_file_name": dgapi_grid_gen_cell_output_file_name, + "point_output_type": "NONE", + } + + expect = gpd.GeoDataFrame.from_features( + [ + { + "type": "Feature", + "properties": {"zone": "51695", "name": "A", "lon": 20.5, "lat": 57.5}, + "geometry": {"type": "Polygon", "coordinates": [[ + [21.189511635794823, 58.2893639588515], + [20.950622414770574, 58.00135094074029], + [21.232988751914608, 57.69437147048094], + [21.74777706813212, 57.674593541688374], + [21.990765226755794, 57.96161815328658], + [21.71493417563983, 58.2694113005297], + [21.189511635794823, 58.2893639588515], + ]]} + }, + { + "type": "Feature", + "properties": {"zone": "51548", "name": "B", "lon": 21.0, "lat": 58.0}, + "geometry": {"type": "Polygon", "coordinates": [[ + [20.430061092474467, 58.01819524792648], + [20.20251122281021, 57.727966938664856], + [20.491335795187357, 57.42178746002092], + [21.001317147943144, 57.405038598901655], + [21.232988751914608, 57.69437147048094], + [20.950622414770574, 58.00135094074029], + [20.430061092474467, 58.01819524792648], + ]]} + }, + ], + columns=result.columns, # ensure ordering matches to allow compare + ) + assert_geodataframe_equal(result, expect) From 76b6df077124220d2fc22d095fdee5042c067067 Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Fri, 31 Oct 2025 19:43:06 -0400 Subject: [PATCH 2/3] forward subset conf inputs --- dggrid4py/dggrid_runner.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dggrid4py/dggrid_runner.py b/dggrid4py/dggrid_runner.py index 5192090..969ae3b 100644 --- a/dggrid4py/dggrid_runner.py +++ b/dggrid4py/dggrid_runner.py @@ -985,6 +985,9 @@ def dgapi_grid_transform( input_extras = self.check_input_extra_fields(conf_extra) subset_conf.update(input_extras) + if subset_conf: + for elem, value in subset_conf.items(): + metafile.append(f"{elem} " + str(value)) # transform output_types output_conf = {} @@ -1078,6 +1081,9 @@ def dgapi_point_value_binning( input_extras = self.check_input_extra_fields(conf_extra) subset_conf.update(input_extras) + if subset_conf: + for elem, value in subset_conf.items(): + metafile.append(f"{elem} " + str(value)) # transform output_types output_conf = {} @@ -1148,6 +1154,9 @@ def dgapi_pres_binning( input_extras = self.check_input_extra_fields(conf_extra) subset_conf.update(input_extras) + if subset_conf: + for elem, value in subset_conf.items(): + metafile.append(f"{elem} " + str(value)) # transform output_types output_conf = {} From 7e1d204cd87b918cd7a39fa0b4549a639b062698 Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Fri, 31 Oct 2025 21:18:35 -0400 Subject: [PATCH 3/3] support additional 'dgapi_pres_binning' arguments + test for 'dgapi_pres_binning' --- dggrid4py/dggrid_runner.py | 16 +++++- tests/test_dggrid.py | 108 +++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/dggrid4py/dggrid_runner.py b/dggrid4py/dggrid_runner.py index 969ae3b..b3f1c80 100644 --- a/dggrid4py/dggrid_runner.py +++ b/dggrid4py/dggrid_runner.py @@ -330,6 +330,7 @@ def get_geo_out(legacy=True, has_gdal=True): "output_cell_label_type": DggsOutputCellLabelTypeT, "output_count": bool, "output_count_field_name": str, + "output_num_classes": bool, "output_delimiter": str, "output_file_name": str, "output_first_seqnum": int, @@ -1158,6 +1159,17 @@ def dgapi_pres_binning( for elem, value in subset_conf.items(): metafile.append(f"{elem} " + str(value)) + # pre-patch elements before integrating them to the metafile + if "output_delimiter" in conf_extra: + delim = conf_extra["output_delimiter"] or " " + delim = f'"{delim}"' if '"' not in delim else delim + conf_extra["output_delimiter"] = delim + if "cell_output_control" in conf_extra and conf_extra["cell_output_control"] not in cell_output_controls: + raise ValueError(f"cell_output_control must be one of: {cell_output_controls}") + for field in ["output_count", "output_num_classes"]: + if field in conf_extra: + conf_extra[field] = str(conf_extra[field]).upper() # bool or string-like bool + # transform output_types output_conf = {} if ( @@ -1165,7 +1177,7 @@ def dgapi_pres_binning( 'output_address_type' in conf_extra.keys() and conf_extra['output_address_type'] in self.output_address_types ): - for elem in filter(lambda x: x.startswith('output_') , conf_extra.keys()): + for elem in filter(lambda x: x.startswith('output_') or x.startswith('cell_output_'), conf_extra.keys()): output_conf[elem] = conf_extra[elem] metafile.append(f"{elem} " + conf_extra[elem]) else: @@ -1175,7 +1187,7 @@ def dgapi_pres_binning( if output_extras: for elem, value in output_extras.items(): output_conf[elem] = conf_extra[elem] - metafile.append(f"{elem} " + str(value)) + metafile.append(f"{elem} {value!s}") result = self.run(metafile) diff --git a/tests/test_dggrid.py b/tests/test_dggrid.py index b1d3fc5..084b737 100644 --- a/tests/test_dggrid.py +++ b/tests/test_dggrid.py @@ -1,7 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- import decimal +import inspect import os +import tempfile import pytest import shapely @@ -141,6 +143,112 @@ def test_dgapi_grid_gen_params(monkeypatch): } +def test_dgapi_pres_binning_params(): + dggs = Dggs( + dggs_type="IGEO7", + aperture=7, + resolution=4, + precision=12, + densification=5, + pole_lon_deg="11.20", + pole_lat_deg=decimal.Decimal("58.282525588538994675786"), + azimuth_deg=0.0, + # geodetic_densify=0.0, # FIXME: crashes DGGRID + ) + + input_data = inspect.cleandoc(""" + -123.28 44.57 49900 Corvallis + -122.87 45.49 42300 Aloha + -122.77 45.43 41700 Tigard + -123.09 44.62 41400 Albany + -122.70 45.41 35700 LakeOswego + -123.02 45.00 32600 Keizer + -123.19 45.21 26800 MacMinnville + -122.60 45.34 26100 OregonCity + -123.32 42.44 23300 GrantsPass + -122.77 45.38 23100 Tualatin + -122.64 45.37 22500 WestLinn + -122.62 45.44 20700 Milwaukie + -121.17 45.60 20600 CitrusPark + -122.86 45.15 20400 Woodburn + -123.36 43.22 20300 Roseburg + """).splitlines() + input_data = [f"{line}\n" for line in input_data] + expect_data = inspect.cleandoc(""" + 021114,1,1 + 021116,1,1 + 014626,1,1 + """).splitlines() + expect_data = [f"{line}\n" for line in expect_data] + + with tempfile.NamedTemporaryFile(suffix=".txt", mode="w") as tmp_shp: + with tempfile.TemporaryDirectory() as tmp_dir: + in_file = tmp_shp.name + tmp_shp.writelines(input_data) + tmp_shp.flush() + tmp_shp.seek(0) + tmp_out = os.path.join(tmp_dir, "out.txt") + result = dggrid.dgapi_pres_binning( + dggs, + { + "input_file_name": in_file, + "input_address_type": "GEO", # required format + }, + **{ + "cell_output_control": "OUTPUT_OCCUPIED", + "output_delimiter": ",", # test that it gets auto-quoted + "output_num_classes": True, + "output_cell_label_type": "OUTPUT_ADDRESS_TYPE", + "output_address_type": "HIERNDX", + "output_hier_ndx_system": "Z7", + "output_hier_ndx_form": "DIGIT_STRING", + "output_file_name": tmp_out, + } + ) + + out_file = result["output_conf"]["output_file_name"] + with open(out_file, "r") as f: + output_data = f.readlines() + assert output_data == expect_data + + assert set(result["metafile"]) == { + "dggrid_operation BIN_POINT_PRESENCE", + "dggs_type IGEO7", + "dggs_proj ISEA", + "dggs_aperture 7", + "dggs_topology HEXAGON", + "dggs_res_spec 4", + "precision 12", + "densification 5", + # "geodetic_densify 0.0", + "dggs_orient_specify_type SPECIFIED", + "dggs_vert0_lon 11.20", + "dggs_vert0_lat 58.282525588538994675786", + "dggs_vert0_azimuth 0.0", + f"input_file_name {in_file}", + "input_address_type GEO", + "cell_output_control OUTPUT_OCCUPIED", + "output_num_classes TRUE", + "output_delimiter \",\"", + "output_cell_label_type OUTPUT_ADDRESS_TYPE", + "output_address_type HIERNDX", + "output_hier_ndx_system Z7", + "output_hier_ndx_form DIGIT_STRING", + f"output_file_name {out_file}", + } + + assert result["output_conf"] == { + "cell_output_control": "OUTPUT_OCCUPIED", + "output_num_classes": "TRUE", + "output_delimiter": '","', + "output_cell_label_type": "OUTPUT_ADDRESS_TYPE", + "output_address_type": "HIERNDX", + "output_hier_ndx_system": "Z7", + "output_hier_ndx_form": "DIGIT_STRING", + "output_file_name": tmp_out, + } + + def test_grid_cell_polygons_for_extent(monkeypatch): metafile = []