From d856b73f3578096037976d6ca9b2276579671c6e Mon Sep 17 00:00:00 2001 From: Scott Davis Date: Mon, 24 Aug 2026 09:40:32 -0600 Subject: [PATCH 01/13] fix(cli): don't add domains to etl output --- cli/src/utrans/etl_common.py | 12 +++++++++--- cli/tests/test_etl_common.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 cli/tests/test_etl_common.py diff --git a/cli/src/utrans/etl_common.py b/cli/src/utrans/etl_common.py index c1deaa7..d529e7e 100644 --- a/cli/src/utrans/etl_common.py +++ b/cli/src/utrans/etl_common.py @@ -131,9 +131,15 @@ def add_missing_template_fields(source_features: str, utrans_roads: str) -> None kwargs: dict[str, object] = {} if field.type == "String": kwargs["field_length"] = field.length - if field.domain: - kwargs["field_domain"] = field.domain - arcpy.management.AddField(source_features, field.name, field.type, **kwargs) + try: + arcpy.management.AddField(source_features, field.name, field.type, **kwargs) + except arcpy.ExecuteError as exc: + details = str(exc) or arcpy.GetMessages(2) + length = f", length={field.length}" if field.type == "String" else "" + raise RuntimeError( + f"Failed to add target field '{field.name}' " + f"(type={field.type}{length}): {details}" + ) from exc def normalize_target_fields(feature_class: str) -> None: diff --git a/cli/tests/test_etl_common.py b/cli/tests/test_etl_common.py new file mode 100644 index 0000000..cab92f5 --- /dev/null +++ b/cli/tests/test_etl_common.py @@ -0,0 +1,34 @@ +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from utrans import etl_common + + +def _field(name, field_type="String", length=None): + return SimpleNamespace(name=name, type=field_type, length=length, domain=None) + + +def _search_cursor(rows): + cursor = Mock() + cursor.__enter__ = Mock(return_value=iter(rows)) + cursor.__exit__ = Mock(return_value=False) + return cursor + + +def test_add_missing_template_fields_reports_failed_field(monkeypatch): + source_fields = [] + target_fields = [_field("ROAD_NAME", length=5)] + add_field = Mock(side_effect=etl_common.arcpy.ExecuteError("invalid length")) + monkeypatch.setattr( + etl_common.arcpy, + "ListFields", + Mock(side_effect=[source_fields, target_fields]), + ) + monkeypatch.setattr(etl_common.arcpy.management, "AddField", add_field) + + with pytest.raises(RuntimeError, match=r"ROAD_NAME.*String.*length=5.*invalid length"): + etl_common.add_missing_template_fields("source", "target") + + add_field.assert_called_once_with("source", "ROAD_NAME", "String", field_length=5) \ No newline at end of file From 819f3025be7076e6d7b83266117d60cb7f65f29d Mon Sep 17 00:00:00 2001 From: Scott Davis Date: Mon, 24 Aug 2026 09:53:31 -0600 Subject: [PATCH 02/13] fix(cli): don't try to insert invalid values that are longer than the fields length --- cli/PROFILES.md | 10 +++++++--- cli/src/utrans/etl.py | 2 +- cli/src/utrans/etl_common.py | 18 +++++++++++++++++- cli/src/utrans/etl_mappers.py | 20 ++++++++++++++++++-- cli/tests/test_etl_common.py | 29 +++++++++++++++++++++++++++-- 5 files changed, 70 insertions(+), 9 deletions(-) diff --git a/cli/PROFILES.md b/cli/PROFILES.md index 8acef3d..a9c6478 100644 --- a/cli/PROFILES.md +++ b/cli/PROFILES.md @@ -21,8 +21,11 @@ The county's five-digit FIPS code written to `COUNTY_L` and `COUNTY_R` during ET A semicolon-delimited list of `UTRANS_FIELD=COUNTY_FIELD` assignments. These map county source fields into the target schema after colliding source fields have been renamed with an underscore. When the UTRANS destination has a coded-value domain, -the assignment is validated against that domain from `--utrans-roads`; invalid values -are appended to `UTRANS_NOTES`. +valid aliases are converted to coded values. Invalid values are preserved and +appended to `UTRANS_NOTES` in all cases. An invalid value is also copied to the +destination field when it fits within that field's length; otherwise the destination +field is left at its default value. Values that exceed the destination field length +are reported with the field and value. ```json "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; NAME=STREETNAME" @@ -40,7 +43,8 @@ names. `TARGET` is `PRIMARY`, `A1`, or `A2`. **Type:** `boolean` | **Optional** — defaults to `false` When `true`, legacy vertical values `1`, `2`, and `3` are translated to `0`, `1`, -and `2` before target-domain validation. +and `2` before domain-value matching. Invalid values are still preserved when they +fit within the destination field length and are always appended to `UTRANS_NOTES`. ### `exclude_if_any` diff --git a/cli/src/utrans/etl.py b/cli/src/utrans/etl.py index 979f55c..bd25139 100644 --- a/cli/src/utrans/etl.py +++ b/cli/src/utrans/etl.py @@ -105,7 +105,7 @@ def run_etl( log(f"Applying {county} county field mappings") apply_mapper(staging_source, profile, utrans_roads) - normalize_target_fields(staging_source) + normalize_target_fields(staging_source, utrans_roads) log("Appending transformed roads to the target schema") arcpy.management.Append(staging_source, staging_output, "NO_TEST") diff --git a/cli/src/utrans/etl_common.py b/cli/src/utrans/etl_common.py index d529e7e..2953024 100644 --- a/cli/src/utrans/etl_common.py +++ b/cli/src/utrans/etl_common.py @@ -142,10 +142,19 @@ def add_missing_template_fields(source_features: str, utrans_roads: str) -> None ) from exc -def normalize_target_fields(feature_class: str) -> None: +def normalize_target_fields(feature_class: str, utrans_roads: str | None = None) -> None: """Normalize target nulls, casing, and legacy name conventions in place.""" names = field_name_map(feature_class) text_fields = [names[name] for name in CORE_TEXT_FIELDS if name in names] + field_lengths: dict[str, int | None] = {} + if utrans_roads is not None: + for field in arcpy.ListFields(utrans_roads): + if field.type != "String" or field.name.upper() not in names: + continue + actual_name = names[field.name.upper()] + if actual_name not in text_fields: + text_fields.append(actual_name) + field_lengths[actual_name.upper()] = field.length numeric_fields = [names[name] for name in CORE_NUMERIC_FIELDS if name in names] fields = text_fields + numeric_fields if not fields: @@ -161,6 +170,13 @@ def normalize_target_fields(feature_class: str) -> None: row[index] = 0 row = _normalize_names(row, fields) + for index, field in enumerate(fields): + length = field_lengths.get(field.upper()) + if length is not None and len(str(row[index])) > length: + raise RuntimeError( + f"Field length exceeded. Field: {field}. " + f"Value: {row[index]!r}" + ) cursor.updateRow(row) diff --git a/cli/src/utrans/etl_mappers.py b/cli/src/utrans/etl_mappers.py index 4f2dbb0..69afdbc 100644 --- a/cli/src/utrans/etl_mappers.py +++ b/cli/src/utrans/etl_mappers.py @@ -95,8 +95,18 @@ def _excluded_values(profile: CountyProfile) -> dict[str, set[str]]: } +def _fits_field_length(value: object, field_lengths: Mapping[str, int | None], field: str) -> bool: + length = field_lengths.get(field.upper()) + return length is None or len(str(value)) <= length + + def apply_mapper(feature_class: str, profile: CountyProfile, utrans_roads: str) -> None: - """Apply a county's root-level profile mapping configuration.""" + """Apply mappings while preserving invalid domain values in the audit notes. + + Valid domain aliases are converted to their coded values. Invalid values are + recorded in UTRANS_NOTES and assigned to the target when they fit its field + length. Field-length validation runs after all mapping and normalization. + """ mappings = _mapping(profile, "field_mappings") if not mappings: raise RuntimeError(f"Profile '{profile.key}' requires field_mappings for ETL.") @@ -136,6 +146,11 @@ def apply_mapper(feature_class: str, profile: CountyProfile, utrans_roads: str) ) ) indexes = {field.upper(): index for index, field in enumerate(fields)} + field_lengths = { + field.name.upper(): field.length + for field in arcpy.ListFields(feature_class) + if field.type == "String" + } domains = domain_values(utrans_roads) posttypes = domains.get("POSTTYPE", {}) vertical_translation = profile.get("translate_vertical_levels", False) @@ -164,12 +179,13 @@ def apply_mapper(feature_class: str, profile: CountyProfile, utrans_roads: str) if resolved is not None: row[target_index] = resolved elif has_value(row[source_index]): - row[target_index] = source_value notes_index = indexes.get("UTRANS_NOTES") if notes_index is not None: row[notes_index] = ( f"{row[notes_index] or ''}{target}: {source_value}; " )[:200] + if _fits_field_length(source_value, field_lengths, target): + row[target_index] = source_value for source, alias in parse_sources: source_index = indexes.get(source.upper()) if source_index is None: diff --git a/cli/tests/test_etl_common.py b/cli/tests/test_etl_common.py index cab92f5..4573227 100644 --- a/cli/tests/test_etl_common.py +++ b/cli/tests/test_etl_common.py @@ -4,13 +4,14 @@ import pytest from utrans import etl_common +from utrans.etl_mappers import _fits_field_length def _field(name, field_type="String", length=None): return SimpleNamespace(name=name, type=field_type, length=length, domain=None) -def _search_cursor(rows): +def _update_cursor(rows): cursor = Mock() cursor.__enter__ = Mock(return_value=iter(rows)) cursor.__exit__ = Mock(return_value=False) @@ -31,4 +32,28 @@ def test_add_missing_template_fields_reports_failed_field(monkeypatch): with pytest.raises(RuntimeError, match=r"ROAD_NAME.*String.*length=5.*invalid length"): etl_common.add_missing_template_fields("source", "target") - add_field.assert_called_once_with("source", "ROAD_NAME", "String", field_length=5) \ No newline at end of file + add_field.assert_called_once_with("source", "ROAD_NAME", "String", field_length=5) + + +def test_normalize_target_fields_reports_overlong_values(monkeypatch): + source_fields = [_field("ONEWAY", length=2)] + target_fields = [_field("ONEWAY", length=1)] + cursor = _update_cursor([["FT"]]) + update_cursor = Mock(return_value=cursor) + monkeypatch.setattr( + etl_common.arcpy, + "ListFields", + Mock(side_effect=[source_fields, target_fields]), + ) + monkeypatch.setattr(etl_common.arcpy.da, "UpdateCursor", update_cursor) + + with pytest.raises(RuntimeError, match=r"ONEWAY.*'FT'"): + etl_common.normalize_target_fields("source", "target") + + +@pytest.mark.parametrize( + ("value", "length", "expected"), + [("F", 1, True), ("FT", 1, False), ("FT", 2, True)], +) +def test_fits_field_length(value, length, expected): + assert _fits_field_length(value, {"ONEWAY": length}, "oneway") is expected \ No newline at end of file From 4ad1294ce9f1011e66def04b67f8f50584a5c94d Mon Sep 17 00:00:00 2001 From: Scott Davis Date: Mon, 24 Aug 2026 10:00:53 -0600 Subject: [PATCH 03/13] fix(cli): fix county name mapping to boundaries data for etl --- cli/src/utrans/etl.py | 4 ++-- cli/tests/test_cli.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cli/src/utrans/etl.py b/cli/src/utrans/etl.py index bd25139..31f500d 100644 --- a/cli/src/utrans/etl.py +++ b/cli/src/utrans/etl.py @@ -30,8 +30,8 @@ def _default_output_name(county: str) -> str: def _county_boundary_name(county: str) -> str: - names = {"boxelder": "Box Elder", "saltlake": "Salt Lake", "sanjuan": "San Juan"} - return names.get(county, county.title()) + names = {"boxelder": "BOX ELDER", "saltlake": "SALT LAKE", "sanjuan": "SAN JUAN"} + return names.get(county, county.upper()) def _unique_name(prefix: str) -> str: diff --git a/cli/tests/test_cli.py b/cli/tests/test_cli.py index 60ed9ba..774e682 100644 --- a/cli/tests/test_cli.py +++ b/cli/tests/test_cli.py @@ -5,6 +5,7 @@ import utrans from utrans import cli +from utrans.etl import _county_boundary_name from utrans.etl_common import direction, parse_full_address, resolve_domain_value from utrans.profiles import load_profiles @@ -61,6 +62,19 @@ def test_etl_help(capsys): assert "--county-boundaries" in captured.out +@pytest.mark.parametrize( + ("county", "expected"), + [ + ("davis", "DAVIS"), + ("boxelder", "BOX ELDER"), + ("saltlake", "SALT LAKE"), + ("sanjuan", "SAN JUAN"), + ], +) +def test_county_boundary_name_is_uppercase(county, expected): + assert _county_boundary_name(county) == expected + + def test_saltlake_profile_replaces_vecc(): profiles = load_profiles() From 93f2e8ceb72706cdd8d44f10dd065d95b9a4e993 Mon Sep 17 00:00:00 2001 From: Scott Davis Date: Mon, 24 Aug 2026 10:05:04 -0600 Subject: [PATCH 04/13] feat(cli): add timing metrics to etl command --- cli/src/utrans/etl.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cli/src/utrans/etl.py b/cli/src/utrans/etl.py index 31f500d..33be914 100644 --- a/cli/src/utrans/etl.py +++ b/cli/src/utrans/etl.py @@ -4,6 +4,7 @@ import argparse import json +import time from datetime import UTC, datetime from pathlib import Path from uuid import uuid4 @@ -173,6 +174,7 @@ def build_parser(prog: str | None = None) -> argparse.ArgumentParser: def main(argv: list[str] | None = None, *, prog: str | None = None) -> int: + start_time = time.time() args = build_parser(prog).parse_args(argv) try: profiles = load_profiles(Path(args.profiles) if args.profiles else None) @@ -190,4 +192,6 @@ def main(argv: list[str] | None = None, *, prog: str | None = None) -> int: except (RuntimeError, TypeError, FileNotFoundError, json.JSONDecodeError) as exc: log(str(exc)) return 2 + elapsed = time.time() - start_time + log(f"Time elapsed: {elapsed:.2f}s") return 0 From cf94198e09773dfa5b31d70f7e8a9e609899e5e0 Mon Sep 17 00:00:00 2001 From: Scott Davis Date: Mon, 24 Aug 2026 11:00:46 -0600 Subject: [PATCH 05/13] chore: address copilot feedback --- cli/tests/test_etl_common.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cli/tests/test_etl_common.py b/cli/tests/test_etl_common.py index 4573227..e2d10eb 100644 --- a/cli/tests/test_etl_common.py +++ b/cli/tests/test_etl_common.py @@ -1,5 +1,5 @@ from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import MagicMock, Mock import pytest @@ -12,9 +12,10 @@ def _field(name, field_type="String", length=None): def _update_cursor(rows): - cursor = Mock() - cursor.__enter__ = Mock(return_value=iter(rows)) - cursor.__exit__ = Mock(return_value=False) + cursor = MagicMock() + cursor.__enter__.return_value = cursor + cursor.__exit__.return_value = False + cursor.__iter__.return_value = iter(rows) return cursor From ea9a79a813cc5c71369b593b68d2a8dae26e3492 Mon Sep 17 00:00:00 2001 From: Scott Davis Date: Mon, 24 Aug 2026 11:05:04 -0600 Subject: [PATCH 06/13] chore(cli): ruff format --- cli/src/utrans/etl_common.py | 7 ++++--- cli/src/utrans/etl_mappers.py | 4 +++- cli/tests/test_etl_common.py | 6 ++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/cli/src/utrans/etl_common.py b/cli/src/utrans/etl_common.py index 2953024..f744d7f 100644 --- a/cli/src/utrans/etl_common.py +++ b/cli/src/utrans/etl_common.py @@ -142,7 +142,9 @@ def add_missing_template_fields(source_features: str, utrans_roads: str) -> None ) from exc -def normalize_target_fields(feature_class: str, utrans_roads: str | None = None) -> None: +def normalize_target_fields( + feature_class: str, utrans_roads: str | None = None +) -> None: """Normalize target nulls, casing, and legacy name conventions in place.""" names = field_name_map(feature_class) text_fields = [names[name] for name in CORE_TEXT_FIELDS if name in names] @@ -174,8 +176,7 @@ def normalize_target_fields(feature_class: str, utrans_roads: str | None = None) length = field_lengths.get(field.upper()) if length is not None and len(str(row[index])) > length: raise RuntimeError( - f"Field length exceeded. Field: {field}. " - f"Value: {row[index]!r}" + f"Field length exceeded. Field: {field}. Value: {row[index]!r}" ) cursor.updateRow(row) diff --git a/cli/src/utrans/etl_mappers.py b/cli/src/utrans/etl_mappers.py index 69afdbc..ef3f4d8 100644 --- a/cli/src/utrans/etl_mappers.py +++ b/cli/src/utrans/etl_mappers.py @@ -95,7 +95,9 @@ def _excluded_values(profile: CountyProfile) -> dict[str, set[str]]: } -def _fits_field_length(value: object, field_lengths: Mapping[str, int | None], field: str) -> bool: +def _fits_field_length( + value: object, field_lengths: Mapping[str, int | None], field: str +) -> bool: length = field_lengths.get(field.upper()) return length is None or len(str(value)) <= length diff --git a/cli/tests/test_etl_common.py b/cli/tests/test_etl_common.py index e2d10eb..5ad5428 100644 --- a/cli/tests/test_etl_common.py +++ b/cli/tests/test_etl_common.py @@ -30,7 +30,9 @@ def test_add_missing_template_fields_reports_failed_field(monkeypatch): ) monkeypatch.setattr(etl_common.arcpy.management, "AddField", add_field) - with pytest.raises(RuntimeError, match=r"ROAD_NAME.*String.*length=5.*invalid length"): + with pytest.raises( + RuntimeError, match=r"ROAD_NAME.*String.*length=5.*invalid length" + ): etl_common.add_missing_template_fields("source", "target") add_field.assert_called_once_with("source", "ROAD_NAME", "String", field_length=5) @@ -57,4 +59,4 @@ def test_normalize_target_fields_reports_overlong_values(monkeypatch): [("F", 1, True), ("FT", 1, False), ("FT", 2, True)], ) def test_fits_field_length(value, length, expected): - assert _fits_field_length(value, {"ONEWAY": length}, "oneway") is expected \ No newline at end of file + assert _fits_field_length(value, {"ONEWAY": length}, "oneway") is expected From 5f8e5d9b2624c8d1d10e90bccc32e76a2086fc7f Mon Sep 17 00:00:00 2001 From: Scott Davis Date: Mon, 24 Aug 2026 11:08:40 -0600 Subject: [PATCH 07/13] chore(cli): sort profiles by key --- cli/src/utrans/profiles.json | 48 ++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/cli/src/utrans/profiles.json b/cli/src/utrans/profiles.json index 3e75fbb..44e0945 100644 --- a/cli/src/utrans/profiles.json +++ b/cli/src/utrans/profiles.json @@ -20,6 +20,12 @@ "field_mappings": "FROMADDR_L=FAL; TOADDR_L=TAL; FROMADDR_R=FAR; TOADDR_R=TAR; NAME=STNAME; A1_NAME=ALIAS; A2_NAME=ALIAS2", "parse_sources": "STNAME=PRIMARY; ALIAS=A1; ALIAS2=A2" }, + "carbon": { + "fips": "49007", + "match_fields": "NAME NAME", + "compare_fields": "PREDIR PREDIR; NAME NAME; POSTTYPE POSTTYPE; POSTDIR POSTDIR; FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R", + "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTTYPE=POSTTYPE_; POSTDIR=POSTDIR_; LOCAL_UID=LOCAL_UID_" + }, "daggett": { "fips": "49009", "match_fields": "S_NAME S_NAME", @@ -27,6 +33,24 @@ "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PRE_DIR; NAME=S_NAME; A2_NAME=ALIAS2; LOCAL_UID=CO_UNIQUE; STATUS=S_STATUS; POSTTYPE=S_TYPE; A2_POSTTYPE=ALIAS2_TYP; DOT_CLASS=CLASS; ONEWAY=ONE_WAY; DOT_SRFTYP=S_SURF2; SPEED_LMT=SPD_LMT", "parse_sources": "ALIAS1=A1; ALIAS2=A2; ACS_ALIAS=A1" }, + "davis": { + "fips": "49011", + "match_fields": "RoadName RoadName", + "compare_fields": "PrefixDirection PrefixDirection; RoadName RoadName; RoadNameType RoadNameType; PostDirection PostDirection; RoadAliasName RoadAliasName; LeftFrom LeftFrom; LeftTo LeftTo; RightFrom RightFrom; RightTo RightTo; MPH MPH", + "required_fields": [ + "PrefixDirection", + "RoadName", + "RoadNameType", + "PostDirection", + "RoadAliasName", + "LeftFrom", + "LeftTo", + "RightFrom", + "RightTo", + "MPH" + ], + "field_mappings": "FROMADDR_L=LeftFrom; TOADDR_L=LeftTo; FROMADDR_R=RightFrom; TOADDR_R=RightTo; PREDIR=PrefixDirection; NAME=RoadName; POSTDIR=PostDirection" + }, "duchesne": { "fips": "49013", "match_fields": "STREETNAME STREETNAME", @@ -42,12 +66,6 @@ "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PRE_DIR; NAME=S_NAME; POSTDIR=SUF_DIR; LOCAL_UID=CO_UNIQUE; STATUS=STATUS_; POSTTYPE=S_TYPE; DOT_CLASS=CLASS; ONEWAY=ONE_WAY; DOT_SRFTYP=S_SURF2; SPEED_LMT=SPD_LMT; A1_POSTTYPE=ALIAS1_TYP; A2_POSTTYPE=ALIAS2_TYP", "exclude_if_any": "S_SURF=400,410,420,430,440" }, - "carbon": { - "fips": "49007", - "match_fields": "NAME NAME", - "compare_fields": "PREDIR PREDIR; NAME NAME; POSTTYPE POSTTYPE; POSTDIR POSTDIR; FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R", - "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTTYPE=POSTTYPE_; POSTDIR=POSTDIR_; LOCAL_UID=LOCAL_UID_" - }, "garfield": { "fips": "49017", "match_fields": "NAME NAME", @@ -180,23 +198,5 @@ "match_fields": "S_NAME S_NAME", "compare_fields": "PREDIR PREDIR; S_NAME S_NAME; STREETTYPE STREETTYPE; SUFDIR SUFDIR; ALIAS ALIAS; ACS_ALIAS ACS_ALIAS; SUFFIX_911 SUFFIX_911; LEFTFROM LEFTFROM; LEFTTO LEFTTO; RIGHTFROM RIGHTFROM; RIGHTTO RIGHTTO", "field_mappings": "FROMADDR_L=LEFTFROM; TOADDR_L=LEFTTO; FROMADDR_R=RIGHTFROM; TOADDR_R=RIGHTTO; PREDIR=PREDIR; NAME=S_NAME; POSTDIR=SUFDIR; LOCAL_UID=S_UNIQUE" - }, - "davis": { - "fips": "49011", - "match_fields": "RoadName RoadName", - "compare_fields": "PrefixDirection PrefixDirection; RoadName RoadName; RoadNameType RoadNameType; PostDirection PostDirection; RoadAliasName RoadAliasName; LeftFrom LeftFrom; LeftTo LeftTo; RightFrom RightFrom; RightTo RightTo; MPH MPH", - "required_fields": [ - "PrefixDirection", - "RoadName", - "RoadNameType", - "PostDirection", - "RoadAliasName", - "LeftFrom", - "LeftTo", - "RightFrom", - "RightTo", - "MPH" - ], - "field_mappings": "FROMADDR_L=LeftFrom; TOADDR_L=LeftTo; FROMADDR_R=RightFrom; TOADDR_R=RightTo; PREDIR=PrefixDirection; NAME=RoadName; POSTDIR=PostDirection" } } From 6c9b97b2ab3e630316dbb75b1d571c4c182d492c Mon Sep 17 00:00:00 2001 From: Scott Davis Date: Mon, 24 Aug 2026 12:55:03 -0600 Subject: [PATCH 08/13] fix(cli): fix fips values --- cli/src/utrans/profiles.json | 60 ++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/cli/src/utrans/profiles.json b/cli/src/utrans/profiles.json index 44e0945..b1487d2 100644 --- a/cli/src/utrans/profiles.json +++ b/cli/src/utrans/profiles.json @@ -1,40 +1,40 @@ { "beaver": { - "fips": "49001", + "fips": "49001 - Beaver", "match_fields": "STREETNAME STREETNAME", "compare_fields": "PREDIR PREDIR; STREETNAME STREETNAME; STREETTYPE STREETTYPE; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD; ALIAS1 ALIAS1; ALIAS1TYP ALIAS1TYP; ACSNAME ACSNAME; ACSSUF ACSSUF; SUFDIR SUFDIR", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; NAME=STREETNAME; POSTDIR=SUFDIR; AN_NAME=ACSNAME; A1_NAME=ALIAS1; A2_NAME=ALIAS2; LOCAL_UID=CO_UNIQUE; STATUS=STATUS_; POSTTYPE=STREETTYPE; DOT_SRFTYP=SURFTYPE; DOT_CLASS=CLASS; VERT_LEVEL=VERTLEVEL", "translate_vertical_levels": true }, "boxelder": { - "fips": "49003", + "fips": "49003 - Box Elder", "match_fields": "S_NAME S_NAME", "compare_fields": "L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD; PRE_DIR PRE_DIR; S_NAME S_NAME; S_TYPE S_TYPE; SUF_DIR SUF_DIR; ACS_ALIAS ACS_ALIAS; ACS_NAME ACS_NAME; ACS_SUF ACS_SUF; ALIAS1 ALIAS1; ALIAS1_TYP ALIAS1_TYP; ALIAS2 ALIAS2; ALIAS2_TYP ALIAS2_TYP", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PRE_DIR; NAME=S_NAME; POSTDIR=SUF_DIR; AN_NAME=ACS_NAME; A1_NAME=ALIAS1; A2_NAME=ALIAS2; LOCAL_UID=CO_UNIQUE; POSTTYPE=S_TYPE; DOT_CLASS=CLASS; VERT_LEVEL=VERTLEVEL", "translate_vertical_levels": true }, "cache": { - "fips": "49005", + "fips": "49005 - Cache", "match_fields": "GEOCODEST GEOCODEST", "compare_fields": "PRE_DIR PRE_DIR; GEOCODEST GEOCODEST; TYPE TYPE; FAL FAL; TAL TAL; FAR FAR; TAR TAR; AliasPreDir AliasPreDir; AliasStreet AliasStreet; AliasType AliasType; AliasSufDir AliasSufDir; SUFFDIR SUFFDIR", "field_mappings": "FROMADDR_L=FAL; TOADDR_L=TAL; FROMADDR_R=FAR; TOADDR_R=TAR; NAME=STNAME; A1_NAME=ALIAS; A2_NAME=ALIAS2", "parse_sources": "STNAME=PRIMARY; ALIAS=A1; ALIAS2=A2" }, "carbon": { - "fips": "49007", + "fips": "49007 - Carbon", "match_fields": "NAME NAME", "compare_fields": "PREDIR PREDIR; NAME NAME; POSTTYPE POSTTYPE; POSTDIR POSTDIR; FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R", "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTTYPE=POSTTYPE_; POSTDIR=POSTDIR_; LOCAL_UID=LOCAL_UID_" }, "daggett": { - "fips": "49009", + "fips": "49009 - Daggett", "match_fields": "S_NAME S_NAME", "compare_fields": "PRE_DIR PRE_DIR; S_NAME S_NAME; S_TYPE S_TYPE; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD; ACS_ALIAS ACS_ALIAS; SUF_DIR SUF_DIR", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PRE_DIR; NAME=S_NAME; A2_NAME=ALIAS2; LOCAL_UID=CO_UNIQUE; STATUS=S_STATUS; POSTTYPE=S_TYPE; A2_POSTTYPE=ALIAS2_TYP; DOT_CLASS=CLASS; ONEWAY=ONE_WAY; DOT_SRFTYP=S_SURF2; SPEED_LMT=SPD_LMT", "parse_sources": "ALIAS1=A1; ALIAS2=A2; ACS_ALIAS=A1" }, "davis": { - "fips": "49011", + "fips": "49011 - Davis", "match_fields": "RoadName RoadName", "compare_fields": "PrefixDirection PrefixDirection; RoadName RoadName; RoadNameType RoadNameType; PostDirection PostDirection; RoadAliasName RoadAliasName; LeftFrom LeftFrom; LeftTo LeftTo; RightFrom RightFrom; RightTo RightTo; MPH MPH", "required_fields": [ @@ -52,7 +52,7 @@ "field_mappings": "FROMADDR_L=LeftFrom; TOADDR_L=LeftTo; FROMADDR_R=RightFrom; TOADDR_R=RightTo; PREDIR=PrefixDirection; NAME=RoadName; POSTDIR=PostDirection" }, "duchesne": { - "fips": "49013", + "fips": "49013 - Duchesne", "match_fields": "STREETNAME STREETNAME", "compare_fields": "PREDIR PREDIR; STREETNAME STREETNAME; STREETTYPE STREETTYPE; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD; ALIAS1 ALIAS1; ALIAS1TYPE ALIAS1TYPE; ALIAS2 ALIAS2; ALIAS2TYPE ALIAS2TYPE; ACSNAME ACSNAME; ACSSUF ACSSUF; SUFDIR SUFDIR", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PREDIR_; NAME=STREETNAME; POSTDIR=SUFDIR; AN_NAME=ACSNAME; A1_NAME=ALIAS1; A2_NAME=ALIAS2; LOCAL_UID=COUNIQUE; POSTTYPE=STREETTYPE; ONEWAY=ONEWAY_; DOT_SRFTYP=SURFTYPE; STATUS=STATUS_; DOT_CLASS=CLASS; A1_POSTTYPE=ALIAS1TYPE; A2_POSTTYPE=ALIAS2TYPE; VERT_LEVEL=VERTLEVEL", @@ -60,20 +60,20 @@ "translate_vertical_levels": true }, "emery": { - "fips": "49015", + "fips": "49015 - Emery", "match_fields": "S_NAME S_NAME", "compare_fields": "PRE_DIR PRE_DIR; S_NAME S_NAME; S_TYPE S_TYPE; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD; ALIAS1 ALIAS1; ALIAS1_TYP ALIAS1_TYP; ALIAS2 ALIAS2; ALIAS2_TYP ALIAS2_TYP; SUF_DIR SUF_DIR", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PRE_DIR; NAME=S_NAME; POSTDIR=SUF_DIR; LOCAL_UID=CO_UNIQUE; STATUS=STATUS_; POSTTYPE=S_TYPE; DOT_CLASS=CLASS; ONEWAY=ONE_WAY; DOT_SRFTYP=S_SURF2; SPEED_LMT=SPD_LMT; A1_POSTTYPE=ALIAS1_TYP; A2_POSTTYPE=ALIAS2_TYP", "exclude_if_any": "S_SURF=400,410,420,430,440" }, "garfield": { - "fips": "49017", + "fips": "49017 - Garfield", "match_fields": "NAME NAME", "compare_fields": "FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R; NAME NAME; PREDIR PREDIR; POSTDIR POSTDIR; POSTTYPE POSTTYPE; AN_NAME AN_NAME; A1_NAME A1_NAME; A2_NAME A2_NAME", "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTDIR=POSTDIR_; AN_NAME=AN_NAME_; A1_NAME=A1_NAME_; A2_NAME=A2_NAME_; LOCAL_UID=LOCAL_UID_; POSTTYPE=POSTTYPE_; A1_POSTTYPE=A1_POSTTYPE_; A2_POSTTYPE=A2_POSTTYPE_; ONEWAY=ONEWAY_; CARTOCODE=CARTOCODE_; STATUS=STATUS_; DOT_SRFTYP=DOT_SRFTYP_; DOT_CLASS=DOT_CLASS_; SPEED_LMT=SPEED_LMT_" }, "grand": { - "fips": "49019", + "fips": "49019 - Grand", "match_fields": "STREETNAME STREETNAME", "compare_fields": "L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD; STREETNAME STREETNAME; PREDIR PREDIR; SUFDIR SUFDIR; STREETTYPE STREETTYPE; ACSALIAS ACSALIAS; ALIAS1 ALIAS1; ALIAS2 ALIAS2", "uppercase_normalize_fields": [ @@ -86,73 +86,79 @@ "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PREDIR_; NAME=STREETNAME; POSTDIR=SUFDIR; AN_NAME=ACSNAME; A1_NAME=ALIAS1; A2_NAME=ALIAS2; LOCAL_UID=COUNIQUE; STATUS=STATUS_; POSTTYPE=STREETTYPE; A1_POSTTYPE=ALIAS1TYPE; A2_POSTTYPE=ALIAS2TYPE; DOT_CLASS=CLASS; ONEWAY=ONEWAY_; DOT_SRFTYP=SURFTYPE; CARTOCODE=CARTOCODE_; SPEED_LMT=SPEED" }, "iron": { - "fips": "49021", + "fips": "49021 - Iron", "match_fields": "STREETNAME STREETNAME", "compare_fields": "PREDIR PREDIR; STREETNAME STREETNAME; STREETTYPE STREETTYPE; SUFDIR SUFDIR; ALIAS1 ALIAS1; ALIAS1TYPE ALIAS1TYPE; ALIAS2 ALIAS2; ALIAS2TYPE ALIAS2TYPE; ACSALIAS ACSALIAS; ACSNAME ACSNAME; ACSSUF ACSSUF; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PREDIR_; NAME=STREETNAME; POSTDIR=SUFDIR; AN_NAME=ACSNAME; A1_NAME=ALIAS1; A2_NAME=ALIAS2; LOCAL_UID=COUNIQUE; POSTTYPE=STREETTYPE; ONEWAY=ONEWAY_; DOT_SRFTYP=SURFTYPE; STATUS=STATUS_; DOT_CLASS=CLASS; VERT_LEVEL=VERTLEVEL", "translate_vertical_levels": true }, "kane": { - "fips": "49025", + "fips": "49025 - Kane", "match_fields": "STREETNAME STREETNAME", "compare_fields": "PREDIR PREDIR; STREETNAME STREETNAME; STREETTYPE STREETTYPE; SUFDIR SUFDIR; ACSALIAS ACSALIAS; ALIAS1 ALIAS1; ALIAS1TYPE ALIAS1TYPE; ALIAS2 ALIAS2; ALIAS2TYPE ALIAS2TYPE; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PREDIR_; NAME=STREETNAME; POSTDIR=SUFDIR; AN_NAME=ACSNAME; A1_NAME=ALIAS1; A2_NAME=ALIAS2; LOCAL_UID=COUNIQUE; STATUS=STATUS_; POSTTYPE=STREETTYPE; A1_POSTTYPE=ALIAS1TYPE; A2_POSTTYPE=ALIAS2TYPE; DOT_CLASS=CLASS; ONEWAY=ONEWAY_; DOT_SRFTYP=SURFTYPE; CARTOCODE=CARTOCODE_; SPEED_LMT=SPEED", "exclude_if_any": "SURFTYPE=400,410,420,430,440,999" }, "millard": { - "fips": "49027", + "fips": "49027 - Millard", "match_fields": "FULLNAME FULLNAME", "compare_fields": "PREDIR PREDIR; FULLNAME FULLNAME; STREETTYPE STREETTYPE; SUFDIR SUFDIR; ALIAS1 ALIAS1; ALIAS1TYPE ALIAS1TYPE; ALIAS2 ALIAS2; ALIAS2TYPE ALIAS2TYPE; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PREDIR_; NAME=FULLNAME; POSTDIR=SUFDIR; A1_NAME=ALIAS1; A2_NAME=ALIAS2; POSTTYPE=STREETTYPE; A1_POSTTYPE=ALIAS1TYPE; A2_POSTTYPE=ALIAS2TYPE; CARTOCODE=CARTOCODE_" }, "morgan": { - "fips": "49029", + "fips": "49029 - Morgan", "match_fields": "FULLNAME FULLNAME", "compare_fields": "FROMLEFT FROMLEFT; TOLEFT TOLEFT; FROMRIGHT FROMRIGHT; TORIGHT TORIGHT; FULLNAME FULLNAME", "field_mappings": "FROMADDR_L=FROMLEFT; TOADDR_L=TOLEFT; FROMADDR_R=FROMRIGHT; TOADDR_R=TORIGHT; FULLNAME=FULLNAME_; ONEWAY=ONEWAYDIR; DOT_CLASS=ROADCLASS; VERT_LEVEL=ROADLEVEL", "parse_sources": "FULLNAME_=PRIMARY" }, "piute": { - "fips": "49031", + "fips": "49031 - Piute", "match_fields": "NAME NAME", "compare_fields": "PREDIR PREDIR; NAME NAME; POSTTYPE POSTTYPE; POSTDIR POSTDIR; FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R", "field_mappings": "FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTTYPE=POSTTYPE_; POSTDIR=POSTDIR_" }, "rich": { - "fips": "49033", + "fips": "49033 - Rich", "match_fields": "S_NAME S_NAME", "compare_fields": "PRE_DIR PRE_DIR; S_NAME S_NAME; S_TYPE S_TYPE; SUF_DIR SUF_DIR; ACS_ALIAS ACS_ALIAS; ALIAS1 ALIAS1; ALIAS1_TYP ALIAS1_TYP; ALIAS2 ALIAS2; ALIAS2_TYP ALIAS2_TYP; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PRE_DIR; NAME=S_NAME; LOCAL_UID=CO_UNIQUE; POSTTYPE=S_TYPE; A1_POSTTYPE=ALIAS1_TYP; A2_POSTTYPE=ALIAS2_TYP; ONEWAY=ONE_WAY; DOT_SRFTYP=S_SURF; SPEED_LMT=SPD_LMT", "parse_sources": "ALIAS1=A1; ALIAS2=A2; ACS_ALIAS=A1" }, "saltlake": { - "fips": "49035", + "fips": "49035 - Salt Lake", "match_fields": "NAME NAME", "compare_fields": "PREDIR PREDIR; NAME NAME; POSTTYPE POSTTYPE; POSTDIR POSTDIR; AN_NAME AN_NAME; AN_POSTDIR AN_POSTDIR; FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R", "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTDIR=POSTDIR_; AN_NAME=AN_NAME_; POSTTYPE=POSTTYPE_; STATUS=STATUS_; ONEWAY=ONEWAY_; DOT_SRFTYP=DOT_SRFTYP_" }, "sanjuan": { - "fips": "49037", + "fips": "49037 - San Juan", "match_fields": "S_NAME S_NAME", "compare_fields": "S_NAME S_NAME", "field_mappings": "NAME=S_NAME; LOCAL_UID=CO_UNIQUE; DOT_CLASS=CLASS", "parse_sources": "S_NAME=PRIMARY" }, + "sanpete": { + "fips": "49039 - Sanpete", + "match_fields": "", + "compare_fields": "", + "field_mappings": "" + }, "sevier": { - "fips": "49041", + "fips": "49041 - Sevier", "match_fields": "S_NAME S_NAME", "compare_fields": "L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD; PRE_DIR PRE_DIR; S_NAME S_NAME; S_TYPE S_TYPE; SUR_DIR SUR_DIR; ALIAS ALIAS", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PRE_DIR; NAME=STREET; LOCAL_UID=CO_UNIQUE; POSTTYPE=S_TYPE", "parse_sources": "STREET=PRIMARY; ALIAS=A1" }, "summit": { - "fips": "49043", + "fips": "49043 - Summit", "match_fields": "STREET STREET", "compare_fields": "PREFIX_DIR PREFIX_DIR; STREET STREET; CLASSIFICA CLASSIFICA; POST_DIR POST_DIR; FROMLEFT FROMLEFT; TOLEFT TOLEFT; FROMRIGHT FROMRIGHT; TORIGHT TORIGHT", "field_mappings": "FROMADDR_L=FROMLEFT; TOADDR_L=TOLEFT; FROMADDR_R=FROMRIGHT; TOADDR_R=TORIGHT; PREDIR=PREFIX_DIR; NAME=STREET; POSTDIR=POST_DIR; POSTTYPE=SUFF_TYPE; ONEWAY=ONEWAY_; STATUS=STATUS_" }, "tooele": { - "fips": "49045", + "fips": "49045 - Tooele", "match_fields": "FULLNAME FULLNAME", "compare_fields": "FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R; FULLNAME FULLNAME; PREDIR PREDIR; A1_NAME A1_NAME; A2_NAME A2_NAME; AN_NAME AN_NAME; STATUS STATUS", "uppercase_normalize_fields": ["FULLNAME", "A1_NAME", "A2_NAME"], @@ -160,7 +166,7 @@ "exclude_if_any": "EXCLUDE=X,P; STATUS_=CONSTRUCTION,PLANNED" }, "uintah": { - "fips": "49047", + "fips": "49047 - Uintah", "match_fields": "S_NAME S_NAME", "compare_fields": "toleft toleft; toright toright; fromleft fromleft; fromright fromright; S_NAME S_NAME; ALIAS1 ALIAS1", "uppercase_normalize_fields": ["S_NAME", "ALIAS1"], @@ -168,33 +174,33 @@ "parse_sources": "S_NAME=PRIMARY; ALIAS1=A1" }, "utah": { - "fips": "49049", + "fips": "49049 - Utah", "match_fields": "ROADNAME ROADNAME", "compare_fields": "ROADPREDIR ROADPREDIR; ROADNAME ROADNAME; ROADTYPE ROADTYPE; ROADPOSTDIR ROADPOSTDIR; ALTROADNAME ALTROADNAME; ALTROADTYPE ALTROADTYPE; ALTROADNAME2 ALTROADNAME2; ALTROADTYPE2 ALTROADTYPE2; FROMLEFT FROMLEFT; TOLEFT TOLEFT; FROMRIGHT FROMRIGHT; TORIGHT TORIGHT", "field_mappings": "FROMADDR_L=FROMLEFT; TOADDR_L=TOLEFT; FROMADDR_R=FROMRIGHT; TOADDR_R=TORIGHT; PREDIR=ROADPREDIR; NAME=ROADNAME" }, "wasatch": { - "fips": "49051", + "fips": "49051 - Wasatch", "match_fields": "FULLNAME FULLNAME", "compare_fields": "FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R; FULLNAME FULLNAME; PREDIR PREDIR; AN_NAME AN_NAME", "uppercase_normalize_fields": ["FULLNAME"], "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTTYPE=POSTTYPE_; POSTDIR=POSTDIR_; AN_NAME=AN_NAME_; LOCAL_UID=LOCAL_UID_" }, "washington": { - "fips": "49053", + "fips": "49053 - Washington", "match_fields": "NAME NAME", "compare_fields": "FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R; PREDIR PREDIR; NAME NAME; POSTTYPE POSTTYPE; POSTDIR POSTDIR; SUFFIXDIR SUFFIXDIR; AN_NAME AN_NAME; AN_POSTDIR AN_POSTDIR; A1_NAME A1_NAME; A1_POSTTYPE A1_POSTTYPE; A2_NAME A2_NAME; A2_POSTTYPE A2_POSTTYPE", "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; AN_NAME=AN_NAME_; A1_NAME=A1_NAME_; A2_NAME=A2_NAME_; LOCAL_UID=LOCAL_UID_" }, "wayne": { - "fips": "49055", + "fips": "49055 - Wayne", "match_fields": "S_NAME S_NAME", "compare_fields": "PRE_DIR PRE_DIR; S_NAME S_NAME; S_TYPE S_TYPE; SUF_DIR SUF_DIR; ACS_ALIAS ACS_ALIAS; ALIAS1 ALIAS1; ALIAS1_TYP ALIAS1_TYP; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PRE_DIR; NAME=S_NAME; LOCAL_UID=CO_UNIQUE; DOT_CLASS=CLASS; ONEWAY=ONE_WAY; DOT_SRFTYP=S_SURF2; VERT_LEVEL=VERTLEVEL", "parse_sources": "S_NAME=PRIMARY; ALIAS1=A1" }, "weber": { - "fips": "49057", + "fips": "49057 - Weber", "match_fields": "S_NAME S_NAME", "compare_fields": "PREDIR PREDIR; S_NAME S_NAME; STREETTYPE STREETTYPE; SUFDIR SUFDIR; ALIAS ALIAS; ACS_ALIAS ACS_ALIAS; SUFFIX_911 SUFFIX_911; LEFTFROM LEFTFROM; LEFTTO LEFTTO; RIGHTFROM RIGHTFROM; RIGHTTO RIGHTTO", "field_mappings": "FROMADDR_L=LEFTFROM; TOADDR_L=LEFTTO; FROMADDR_R=RIGHTFROM; TOADDR_R=RIGHTTO; PREDIR=PREDIR; NAME=S_NAME; POSTDIR=SUFDIR; LOCAL_UID=S_UNIQUE" From b90b99e650c8b631aa6b32824a57d7c4f7c6eabd Mon Sep 17 00:00:00 2001 From: Scott Davis Date: Mon, 24 Aug 2026 14:47:28 -0600 Subject: [PATCH 09/13] feat(cli): implement value_mappings, rules, and custom_handler profile configs For implementing more county-specific etl logic --- .vscode/settings.json | 12 +++ cli/PROFILES.md | 63 ++++++++++++++ cli/src/utrans/etl_common.py | 4 +- cli/src/utrans/etl_handlers.py | 115 +++++++++++++++++++++++++ cli/src/utrans/etl_mappers.py | 68 +++++++++++++++ cli/src/utrans/etl_rules.py | 53 ++++++++++++ cli/src/utrans/profiles.json | 150 +++++++++++++++++---------------- cli/tests/test_cli.py | 2 +- cli/tests/test_etl_mappers.py | 123 +++++++++++++++++++++++++++ 9 files changed, 513 insertions(+), 77 deletions(-) create mode 100644 cli/src/utrans/etl_handlers.py create mode 100644 cli/src/utrans/etl_rules.py create mode 100644 cli/tests/test_etl_mappers.py diff --git a/.vscode/settings.json b/.vscode/settings.json index aea395f..997bb73 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -42,6 +42,18 @@ "/^dotnet build add-in\\\\UGRC\\.UtransTools\\\\UGRC\\.UtransTools\\.csproj --no-restore$/": { "approve": true, "matchCommandLine": true + }, + "/^conda activate utrans-tools$/": { + "approve": true, + "matchCommandLine": true + }, + "/^conda activate utrans-tools; pytest/": { + "approve": true, + "matchCommandLine": true + }, + "/^conda activate utrans-tools; ruff/": { + "approve": true, + "matchCommandLine": true } }, "[csharp]": { diff --git a/cli/PROFILES.md b/cli/PROFILES.md index a9c6478..823e078 100644 --- a/cli/PROFILES.md +++ b/cli/PROFILES.md @@ -31,6 +31,69 @@ are reported with the field and value. "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; NAME=STREETNAME" ``` +### `value_mappings` + +**Type:** `object` | **Optional** — defaults to `{}` + +Per-target mappings for source values that do not match a UTRANS coded-value +domain code or description. Each target field contains an object mapping the +source value to the desired target value. Field names and source values are +matched case-insensitively after surrounding whitespace is removed. The mapped +target value is then resolved against the destination coded-value domain, so it +can be either a coded value or a domain description. + +```json +"value_mappings": { + "ONEWAY": { + "ONE DIRECTION": "Y", + "TWO WAY": "N" + }, + "DOT_CLASS": { + "LOCAL ROAD": "L" + } +} +``` + +Mappings run after `field_mappings` selects a source field and before normal +destination-domain matching. If no configured mapping or valid domain match is +found, the original source value is handled as an invalid value: it is appended +to `UTRANS_NOTES` and copied only when it fits the target field length. + +### `rules` + +**Type:** `string[]` | **Optional** — defaults to `[]` + +An ordered list of supported post-mapping transformations. Rules run after field +and full-address mappings, but before the feature is normalized and appended. Unknown +or duplicate names are rejected. + +```json +"rules": ["remove_postdir_if_alpha", "remove_posttype_if_numeric"] +``` + +Supported rules: + +- `remove_postdir_if_alpha`: clears `POSTDIR` when `NAME` begins with a letter. +- `remove_posttype_if_numeric`: clears `POSTTYPE` when `NAME` begins with a digit. + +### `custom_handler` + +**Type:** `string` | **Optional** + +Selects a built-in, county-specific transformation for behavior that requires +multiple fields or conditional assignments. Handler names are resolved from the +CLI's fixed registry; profile files cannot name Python modules or functions. +Unknown handler names are rejected before ETL updates begin. + +```json +"custom_handler": "utah_road_names" +``` + +Currently supported handlers: + +- `utah_road_names`: preserves Utah County's legacy handling of numeric primary + and alternate road names ending in a cardinal direction. + ### `parse_sources` **Type:** `string` | **Optional** diff --git a/cli/src/utrans/etl_common.py b/cli/src/utrans/etl_common.py index f744d7f..09b7d33 100644 --- a/cli/src/utrans/etl_common.py +++ b/cli/src/utrans/etl_common.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from dataclasses import dataclass from pathlib import Path @@ -105,7 +105,7 @@ def domain_values(utrans_roads: str) -> dict[str, dict[str, str]]: return resolved -def resolve_domain_value(value: object, values: dict[str, str]) -> str | None: +def resolve_domain_value(value: object, values: Mapping[str, str]) -> str | None: if not has_value(value): return "" normalized = str(value).strip().upper() diff --git a/cli/src/utrans/etl_handlers.py b/cli/src/utrans/etl_handlers.py new file mode 100644 index 0000000..c3df0e7 --- /dev/null +++ b/cli/src/utrans/etl_handlers.py @@ -0,0 +1,115 @@ +"""Registered county-specific ETL transformations. + +Profiles may select only handlers defined in this module; they cannot import +arbitrary code from a custom profiles file. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass + +from utrans.etl_common import direction, has_value, resolve_domain_value +from utrans.profiles import CountyProfile + +RowHandler = Callable[ + [list[object], Mapping[str, int], Mapping[str, Mapping[str, str]]], None +] + + +@dataclass(frozen=True) +class Handler: + required_fields: tuple[str, ...] + apply: RowHandler + + +def _value(row: list[object], indexes: Mapping[str, int], field: str) -> object: + index = indexes.get(field.upper()) + return row[index] if index is not None else "" + + +def _set( + row: list[object], indexes: Mapping[str, int], field: str, value: object +) -> None: + index = indexes.get(field.upper()) + if index is not None: + row[index] = value + + +def _numeric_name_and_direction(value: object) -> tuple[str, str] | None: + if not has_value(value): + return None + parts = str(value).strip().upper().split() + if len(parts) < 2 or not parts[0].isdigit(): + return None + postdir = direction(parts[-1]) + return (parts[0], postdir) if postdir else None + + +def _utah_road_names( + row: list[object], + indexes: Mapping[str, int], + domains: Mapping[str, Mapping[str, str]], +) -> None: + """Preserve Utah County's legacy numeric primary and alternate road parsing.""" + primary = _numeric_name_and_direction(_value(row, indexes, "ROADNAME")) + if primary is not None: + _set(row, indexes, "NAME", primary[0]) + _set(row, indexes, "POSTDIR", primary[1]) + _set(row, indexes, "POSTTYPE", "") + + numeric_alias: tuple[str, str] | None = None + for source, target in (("ALTROADNAME2", "A2_NAME"), ("ALTROADNAME", "A1_NAME")): + parsed = _numeric_name_and_direction(_value(row, indexes, source)) + if parsed is None: + continue + numeric_alias = parsed + _set(row, indexes, target, "") + _set(row, indexes, "A1_POSTTYPE", "") + if numeric_alias is not None: + _set(row, indexes, "AN_NAME", numeric_alias[0]) + _set(row, indexes, "AN_POSTDIR", numeric_alias[1]) + + alias_name = _value(row, indexes, "A1_NAME") + if not has_value(alias_name): + _set(row, indexes, "A1_POSTTYPE", "") + return + alias_type = _value(row, indexes, "ALTROADTYPE") + first_word = ( + str(alias_type).strip().split(maxsplit=1)[0] if has_value(alias_type) else "" + ) + resolved = resolve_domain_value(first_word, domains.get("A1_POSTTYPE", {})) + if resolved is not None: + _set(row, indexes, "A1_POSTTYPE", resolved) + + +HANDLERS: Mapping[str, Handler] = { + "utah_road_names": Handler( + required_fields=( + "ROADNAME", + "ALTROADNAME", + "ALTROADTYPE", + "ALTROADNAME2", + ), + apply=_utah_road_names, + ) +} + + +def profile_handler(profile: CountyProfile) -> Handler | None: + """Resolve an optional handler name from the closed registry.""" + value = profile.get("custom_handler") + if value is None or value == "": + return None + if not isinstance(value, str): + raise TypeError( + f"Profile '{profile.key}' setting 'custom_handler' must be a string." + ) + try: + return HANDLERS[value] + except KeyError as exc: + names = ", ".join(sorted(HANDLERS)) + raise ValueError( + f"Profile '{profile.key}' has unsupported custom_handler '{value}'. " + f"Supported handlers: {names}." + ) from exc diff --git a/cli/src/utrans/etl_mappers.py b/cli/src/utrans/etl_mappers.py index ef3f4d8..66d789b 100644 --- a/cli/src/utrans/etl_mappers.py +++ b/cli/src/utrans/etl_mappers.py @@ -13,6 +13,8 @@ parse_full_address, resolve_domain_value, ) +from utrans.etl_handlers import profile_handler +from utrans.etl_rules import apply_rules, profile_rules from utrans.profiles import CountyProfile @@ -69,6 +71,61 @@ def _parse_sources(profile: CountyProfile) -> tuple[tuple[str, str], ...]: return tuple(result) +def _value_mappings(profile: CountyProfile) -> dict[str, dict[str, str]]: + """Return configured source-value to target-value mappings by target field.""" + value = profile.get("value_mappings", {}) + if not isinstance(value, Mapping): + raise TypeError( + f"Profile '{profile.key}' setting 'value_mappings' must be an object." + ) + mappings: dict[str, dict[str, str]] = {} + for target, values in value.items(): + if not isinstance(target, str) or not isinstance(values, Mapping): + raise TypeError( + f"Profile '{profile.key}' value_mappings must map field names to objects." + ) + if not all( + isinstance(source, str) and isinstance(destination, str) + for source, destination in values.items() + ): + raise TypeError( + f"Profile '{profile.key}' value_mappings for '{target}' must map strings to strings." + ) + mappings[target.upper()] = { + source.strip().upper(): destination.strip() + for source, destination in values.items() + } + return mappings + + +def _validate_value_mappings( + profile: CountyProfile, + value_mappings: Mapping[str, Mapping[str, str]], + names: Mapping[str, str], + domains: Mapping[str, Mapping[str, str]], +) -> None: + for target, values in value_mappings.items(): + if target not in names: + raise ValueError( + f"Profile '{profile.key}' value_mappings references unknown target field " + f"'{target}'." + ) + domain = domains.get(target) + if domain is None: + continue + invalid = [ + value + for value in values.values() + if resolve_domain_value(value, domain) is None + ] + if invalid: + rendered = ", ".join(repr(value) for value in invalid) + raise ValueError( + f"Profile '{profile.key}' value_mappings for '{target}' contains values " + f"outside the target domain: {rendered}." + ) + + def _excluded_values(profile: CountyProfile) -> dict[str, set[str]]: value = profile.get("exclude_if_any", "") if isinstance(value, str): @@ -113,6 +170,9 @@ def apply_mapper(feature_class: str, profile: CountyProfile, utrans_roads: str) if not mappings: raise RuntimeError(f"Profile '{profile.key}' requires field_mappings for ETL.") parse_sources = _parse_sources(profile) + value_mappings = _value_mappings(profile) + rules = profile_rules(profile) + handler = profile_handler(profile) names = field_name_map(feature_class) mappings = { **{ @@ -139,6 +199,7 @@ def apply_mapper(feature_class: str, profile: CountyProfile, utrans_roads: str) source_fields = [ *mappings.values(), *(source for source, _ in parse_sources), + *(handler.required_fields if handler is not None else ()), ] fields = list( dict.fromkeys( @@ -154,6 +215,7 @@ def apply_mapper(feature_class: str, profile: CountyProfile, utrans_roads: str) if field.type == "String" } domains = domain_values(utrans_roads) + _validate_value_mappings(profile, value_mappings, names, domains) posttypes = domains.get("POSTTYPE", {}) vertical_translation = profile.get("translate_vertical_levels", False) if not isinstance(vertical_translation, bool): @@ -177,6 +239,9 @@ def apply_mapper(feature_class: str, profile: CountyProfile, utrans_roads: str) source_value = {"1": "0", "2": "1", "3": "2"}.get( str(source_value).strip(), source_value ) + source_value = value_mappings.get(target.upper(), {}).get( + str(source_value).strip().upper(), source_value + ) resolved = resolve_domain_value(source_value, values) if resolved is not None: row[target_index] = resolved @@ -217,6 +282,9 @@ def apply_mapper(feature_class: str, profile: CountyProfile, utrans_roads: str) index = indexes.get(f"{prefix}{suffix}".upper()) if index is not None: row[index] = value + apply_rules(row, indexes, rules) + if handler is not None: + handler.apply(row, indexes, domains) for field, value in ( ("COUNTY_L", profile.require("fips")), ("COUNTY_R", profile.require("fips")), diff --git a/cli/src/utrans/etl_rules.py b/cli/src/utrans/etl_rules.py new file mode 100644 index 0000000..4f19462 --- /dev/null +++ b/cli/src/utrans/etl_rules.py @@ -0,0 +1,53 @@ +"""Supported declarative post-mapping rules for county ETL profiles.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence + +from utrans.etl_common import has_value +from utrans.profiles import CountyProfile + +SUPPORTED_RULES = frozenset({"remove_postdir_if_alpha", "remove_posttype_if_numeric"}) + + +def profile_rules(profile: CountyProfile) -> tuple[str, ...]: + """Validate and return the ordered post-mapping rules for a profile.""" + value = profile.get("rules", []) + if ( + not isinstance(value, Sequence) + or isinstance(value, str) + or not all(isinstance(rule, str) for rule in value) + ): + raise TypeError( + f"Profile '{profile.key}' setting 'rules' must be a list of strings." + ) + unknown = set(value) - SUPPORTED_RULES + if unknown: + names = ", ".join(sorted(unknown)) + raise ValueError( + f"Profile '{profile.key}' has unsupported ETL rule(s): {names}." + ) + if len(set(value)) != len(value): + raise ValueError( + f"Profile '{profile.key}' setting 'rules' cannot contain duplicates." + ) + return tuple(value) + + +def apply_rules( + row: list[object], indexes: Mapping[str, int], rules: Sequence[str] +) -> None: + """Apply supported post-mapping rules to a cursor row in profile order.""" + name_index = indexes.get("NAME") + if name_index is None or not has_value(row[name_index]): + return + name = str(row[name_index]).strip() + for rule in rules: + if rule == "remove_postdir_if_alpha" and name[0].isalpha(): + postdir_index = indexes.get("POSTDIR") + if postdir_index is not None: + row[postdir_index] = "" + elif rule == "remove_posttype_if_numeric" and name[0].isdigit(): + posttype_index = indexes.get("POSTTYPE") + if posttype_index is not None: + row[posttype_index] = "" diff --git a/cli/src/utrans/profiles.json b/cli/src/utrans/profiles.json index b1487d2..fe4676d 100644 --- a/cli/src/utrans/profiles.json +++ b/cli/src/utrans/profiles.json @@ -1,42 +1,43 @@ { "beaver": { - "fips": "49001 - Beaver", - "match_fields": "STREETNAME STREETNAME", "compare_fields": "PREDIR PREDIR; STREETNAME STREETNAME; STREETTYPE STREETTYPE; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD; ALIAS1 ALIAS1; ALIAS1TYP ALIAS1TYP; ACSNAME ACSNAME; ACSSUF ACSSUF; SUFDIR SUFDIR", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; NAME=STREETNAME; POSTDIR=SUFDIR; AN_NAME=ACSNAME; A1_NAME=ALIAS1; A2_NAME=ALIAS2; LOCAL_UID=CO_UNIQUE; STATUS=STATUS_; POSTTYPE=STREETTYPE; DOT_SRFTYP=SURFTYPE; DOT_CLASS=CLASS; VERT_LEVEL=VERTLEVEL", + "fips": "49001 - Beaver", + "match_fields": "STREETNAME STREETNAME", "translate_vertical_levels": true }, "boxelder": { - "fips": "49003 - Box Elder", - "match_fields": "S_NAME S_NAME", "compare_fields": "L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD; PRE_DIR PRE_DIR; S_NAME S_NAME; S_TYPE S_TYPE; SUF_DIR SUF_DIR; ACS_ALIAS ACS_ALIAS; ACS_NAME ACS_NAME; ACS_SUF ACS_SUF; ALIAS1 ALIAS1; ALIAS1_TYP ALIAS1_TYP; ALIAS2 ALIAS2; ALIAS2_TYP ALIAS2_TYP", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PRE_DIR; NAME=S_NAME; POSTDIR=SUF_DIR; AN_NAME=ACS_NAME; A1_NAME=ALIAS1; A2_NAME=ALIAS2; LOCAL_UID=CO_UNIQUE; POSTTYPE=S_TYPE; DOT_CLASS=CLASS; VERT_LEVEL=VERTLEVEL", + "fips": "49003 - Box Elder", + "match_fields": "S_NAME S_NAME", "translate_vertical_levels": true }, "cache": { - "fips": "49005 - Cache", - "match_fields": "GEOCODEST GEOCODEST", "compare_fields": "PRE_DIR PRE_DIR; GEOCODEST GEOCODEST; TYPE TYPE; FAL FAL; TAL TAL; FAR FAR; TAR TAR; AliasPreDir AliasPreDir; AliasStreet AliasStreet; AliasType AliasType; AliasSufDir AliasSufDir; SUFFDIR SUFFDIR", "field_mappings": "FROMADDR_L=FAL; TOADDR_L=TAL; FROMADDR_R=FAR; TOADDR_R=TAR; NAME=STNAME; A1_NAME=ALIAS; A2_NAME=ALIAS2", + "fips": "49005 - Cache", + "match_fields": "GEOCODEST GEOCODEST", "parse_sources": "STNAME=PRIMARY; ALIAS=A1; ALIAS2=A2" }, "carbon": { - "fips": "49007 - Carbon", - "match_fields": "NAME NAME", "compare_fields": "PREDIR PREDIR; NAME NAME; POSTTYPE POSTTYPE; POSTDIR POSTDIR; FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R", - "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTTYPE=POSTTYPE_; POSTDIR=POSTDIR_; LOCAL_UID=LOCAL_UID_" + "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTTYPE=POSTTYPE_; POSTDIR=POSTDIR_; LOCAL_UID=LOCAL_UID_", + "fips": "49007 - Carbon", + "match_fields": "NAME NAME" }, "daggett": { - "fips": "49009 - Daggett", - "match_fields": "S_NAME S_NAME", "compare_fields": "PRE_DIR PRE_DIR; S_NAME S_NAME; S_TYPE S_TYPE; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD; ACS_ALIAS ACS_ALIAS; SUF_DIR SUF_DIR", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PRE_DIR; NAME=S_NAME; A2_NAME=ALIAS2; LOCAL_UID=CO_UNIQUE; STATUS=S_STATUS; POSTTYPE=S_TYPE; A2_POSTTYPE=ALIAS2_TYP; DOT_CLASS=CLASS; ONEWAY=ONE_WAY; DOT_SRFTYP=S_SURF2; SPEED_LMT=SPD_LMT", + "fips": "49009 - Daggett", + "match_fields": "S_NAME S_NAME", "parse_sources": "ALIAS1=A1; ALIAS2=A2; ACS_ALIAS=A1" }, "davis": { + "compare_fields": "PrefixDirection PrefixDirection; RoadName RoadName; RoadNameType RoadNameType; PostDirection PostDirection; RoadAliasName RoadAliasName; LeftFrom LeftFrom; LeftTo LeftTo; RightFrom RightFrom; RightTo RightTo; MPH MPH", + "field_mappings": "FROMADDR_L=LeftFrom; TOADDR_L=LeftTo; FROMADDR_R=RightFrom; TOADDR_R=RightTo; PREDIR=PrefixDirection; NAME=RoadName; POSTDIR=PostDirection", "fips": "49011 - Davis", "match_fields": "RoadName RoadName", - "compare_fields": "PrefixDirection PrefixDirection; RoadName RoadName; RoadNameType RoadNameType; PostDirection PostDirection; RoadAliasName RoadAliasName; LeftFrom LeftFrom; LeftTo LeftTo; RightFrom RightFrom; RightTo RightTo; MPH MPH", "required_fields": [ "PrefixDirection", "RoadName", @@ -48,161 +49,162 @@ "RightFrom", "RightTo", "MPH" - ], - "field_mappings": "FROMADDR_L=LeftFrom; TOADDR_L=LeftTo; FROMADDR_R=RightFrom; TOADDR_R=RightTo; PREDIR=PrefixDirection; NAME=RoadName; POSTDIR=PostDirection" + ] }, "duchesne": { - "fips": "49013 - Duchesne", - "match_fields": "STREETNAME STREETNAME", "compare_fields": "PREDIR PREDIR; STREETNAME STREETNAME; STREETTYPE STREETTYPE; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD; ALIAS1 ALIAS1; ALIAS1TYPE ALIAS1TYPE; ALIAS2 ALIAS2; ALIAS2TYPE ALIAS2TYPE; ACSNAME ACSNAME; ACSSUF ACSSUF; SUFDIR SUFDIR", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PREDIR_; NAME=STREETNAME; POSTDIR=SUFDIR; AN_NAME=ACSNAME; A1_NAME=ALIAS1; A2_NAME=ALIAS2; LOCAL_UID=COUNIQUE; POSTTYPE=STREETTYPE; ONEWAY=ONEWAY_; DOT_SRFTYP=SURFTYPE; STATUS=STATUS_; DOT_CLASS=CLASS; A1_POSTTYPE=ALIAS1TYPE; A2_POSTTYPE=ALIAS2TYPE; VERT_LEVEL=VERTLEVEL", + "fips": "49013 - Duchesne", + "match_fields": "STREETNAME STREETNAME", "parse_sources": "ALIAS1=A1; ALIAS2=A2", "translate_vertical_levels": true }, "emery": { - "fips": "49015 - Emery", - "match_fields": "S_NAME S_NAME", "compare_fields": "PRE_DIR PRE_DIR; S_NAME S_NAME; S_TYPE S_TYPE; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD; ALIAS1 ALIAS1; ALIAS1_TYP ALIAS1_TYP; ALIAS2 ALIAS2; ALIAS2_TYP ALIAS2_TYP; SUF_DIR SUF_DIR", + "exclude_if_any": "S_SURF=400,410,420,430,440", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PRE_DIR; NAME=S_NAME; POSTDIR=SUF_DIR; LOCAL_UID=CO_UNIQUE; STATUS=STATUS_; POSTTYPE=S_TYPE; DOT_CLASS=CLASS; ONEWAY=ONE_WAY; DOT_SRFTYP=S_SURF2; SPEED_LMT=SPD_LMT; A1_POSTTYPE=ALIAS1_TYP; A2_POSTTYPE=ALIAS2_TYP", - "exclude_if_any": "S_SURF=400,410,420,430,440" + "fips": "49015 - Emery", + "match_fields": "S_NAME S_NAME" }, "garfield": { - "fips": "49017 - Garfield", - "match_fields": "NAME NAME", "compare_fields": "FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R; NAME NAME; PREDIR PREDIR; POSTDIR POSTDIR; POSTTYPE POSTTYPE; AN_NAME AN_NAME; A1_NAME A1_NAME; A2_NAME A2_NAME", - "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTDIR=POSTDIR_; AN_NAME=AN_NAME_; A1_NAME=A1_NAME_; A2_NAME=A2_NAME_; LOCAL_UID=LOCAL_UID_; POSTTYPE=POSTTYPE_; A1_POSTTYPE=A1_POSTTYPE_; A2_POSTTYPE=A2_POSTTYPE_; ONEWAY=ONEWAY_; CARTOCODE=CARTOCODE_; STATUS=STATUS_; DOT_SRFTYP=DOT_SRFTYP_; DOT_CLASS=DOT_CLASS_; SPEED_LMT=SPEED_LMT_" + "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTDIR=POSTDIR_; AN_NAME=AN_NAME_; A1_NAME=A1_NAME_; A2_NAME=A2_NAME_; LOCAL_UID=LOCAL_UID_; POSTTYPE=POSTTYPE_; A1_POSTTYPE=A1_POSTTYPE_; A2_POSTTYPE=A2_POSTTYPE_; ONEWAY=ONEWAY_; CARTOCODE=CARTOCODE_; STATUS=STATUS_; DOT_SRFTYP=DOT_SRFTYP_; DOT_CLASS=DOT_CLASS_; SPEED_LMT=SPEED_LMT_", + "fips": "49017 - Garfield", + "match_fields": "NAME NAME" }, "grand": { + "compare_fields": "L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD; STREETNAME STREETNAME; PREDIR PREDIR; SUFDIR SUFDIR; STREETTYPE STREETTYPE; ACSALIAS ACSALIAS; ALIAS1 ALIAS1; ALIAS2 ALIAS2", + "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PREDIR_; NAME=STREETNAME; POSTDIR=SUFDIR; AN_NAME=ACSNAME; A1_NAME=ALIAS1; A2_NAME=ALIAS2; LOCAL_UID=COUNIQUE; STATUS=STATUS_; POSTTYPE=STREETTYPE; A1_POSTTYPE=ALIAS1TYPE; A2_POSTTYPE=ALIAS2TYPE; DOT_CLASS=CLASS; ONEWAY=ONEWAY_; DOT_SRFTYP=SURFTYPE; CARTOCODE=CARTOCODE_; SPEED_LMT=SPEED", "fips": "49019 - Grand", "match_fields": "STREETNAME STREETNAME", - "compare_fields": "L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD; STREETNAME STREETNAME; PREDIR PREDIR; SUFDIR SUFDIR; STREETTYPE STREETTYPE; ACSALIAS ACSALIAS; ALIAS1 ALIAS1; ALIAS2 ALIAS2", "uppercase_normalize_fields": [ "STREETNAME", "STREETTYPE", "ACSALIAS", "ALIAS1", "ALIAS2" - ], - "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PREDIR_; NAME=STREETNAME; POSTDIR=SUFDIR; AN_NAME=ACSNAME; A1_NAME=ALIAS1; A2_NAME=ALIAS2; LOCAL_UID=COUNIQUE; STATUS=STATUS_; POSTTYPE=STREETTYPE; A1_POSTTYPE=ALIAS1TYPE; A2_POSTTYPE=ALIAS2TYPE; DOT_CLASS=CLASS; ONEWAY=ONEWAY_; DOT_SRFTYP=SURFTYPE; CARTOCODE=CARTOCODE_; SPEED_LMT=SPEED" + ] }, "iron": { - "fips": "49021 - Iron", - "match_fields": "STREETNAME STREETNAME", "compare_fields": "PREDIR PREDIR; STREETNAME STREETNAME; STREETTYPE STREETTYPE; SUFDIR SUFDIR; ALIAS1 ALIAS1; ALIAS1TYPE ALIAS1TYPE; ALIAS2 ALIAS2; ALIAS2TYPE ALIAS2TYPE; ACSALIAS ACSALIAS; ACSNAME ACSNAME; ACSSUF ACSSUF; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PREDIR_; NAME=STREETNAME; POSTDIR=SUFDIR; AN_NAME=ACSNAME; A1_NAME=ALIAS1; A2_NAME=ALIAS2; LOCAL_UID=COUNIQUE; POSTTYPE=STREETTYPE; ONEWAY=ONEWAY_; DOT_SRFTYP=SURFTYPE; STATUS=STATUS_; DOT_CLASS=CLASS; VERT_LEVEL=VERTLEVEL", + "fips": "49021 - Iron", + "match_fields": "STREETNAME STREETNAME", "translate_vertical_levels": true }, "kane": { - "fips": "49025 - Kane", - "match_fields": "STREETNAME STREETNAME", "compare_fields": "PREDIR PREDIR; STREETNAME STREETNAME; STREETTYPE STREETTYPE; SUFDIR SUFDIR; ACSALIAS ACSALIAS; ALIAS1 ALIAS1; ALIAS1TYPE ALIAS1TYPE; ALIAS2 ALIAS2; ALIAS2TYPE ALIAS2TYPE; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD", + "exclude_if_any": "SURFTYPE=400,410,420,430,440,999", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PREDIR_; NAME=STREETNAME; POSTDIR=SUFDIR; AN_NAME=ACSNAME; A1_NAME=ALIAS1; A2_NAME=ALIAS2; LOCAL_UID=COUNIQUE; STATUS=STATUS_; POSTTYPE=STREETTYPE; A1_POSTTYPE=ALIAS1TYPE; A2_POSTTYPE=ALIAS2TYPE; DOT_CLASS=CLASS; ONEWAY=ONEWAY_; DOT_SRFTYP=SURFTYPE; CARTOCODE=CARTOCODE_; SPEED_LMT=SPEED", - "exclude_if_any": "SURFTYPE=400,410,420,430,440,999" + "fips": "49025 - Kane", + "match_fields": "STREETNAME STREETNAME" }, "millard": { - "fips": "49027 - Millard", - "match_fields": "FULLNAME FULLNAME", "compare_fields": "PREDIR PREDIR; FULLNAME FULLNAME; STREETTYPE STREETTYPE; SUFDIR SUFDIR; ALIAS1 ALIAS1; ALIAS1TYPE ALIAS1TYPE; ALIAS2 ALIAS2; ALIAS2TYPE ALIAS2TYPE; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD", - "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PREDIR_; NAME=FULLNAME; POSTDIR=SUFDIR; A1_NAME=ALIAS1; A2_NAME=ALIAS2; POSTTYPE=STREETTYPE; A1_POSTTYPE=ALIAS1TYPE; A2_POSTTYPE=ALIAS2TYPE; CARTOCODE=CARTOCODE_" + "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PREDIR_; NAME=FULLNAME; POSTDIR=SUFDIR; A1_NAME=ALIAS1; A2_NAME=ALIAS2; POSTTYPE=STREETTYPE; A1_POSTTYPE=ALIAS1TYPE; A2_POSTTYPE=ALIAS2TYPE; CARTOCODE=CARTOCODE_", + "fips": "49027 - Millard", + "match_fields": "FULLNAME FULLNAME" }, "morgan": { - "fips": "49029 - Morgan", - "match_fields": "FULLNAME FULLNAME", "compare_fields": "FROMLEFT FROMLEFT; TOLEFT TOLEFT; FROMRIGHT FROMRIGHT; TORIGHT TORIGHT; FULLNAME FULLNAME", "field_mappings": "FROMADDR_L=FROMLEFT; TOADDR_L=TOLEFT; FROMADDR_R=FROMRIGHT; TOADDR_R=TORIGHT; FULLNAME=FULLNAME_; ONEWAY=ONEWAYDIR; DOT_CLASS=ROADCLASS; VERT_LEVEL=ROADLEVEL", + "fips": "49029 - Morgan", + "match_fields": "FULLNAME FULLNAME", "parse_sources": "FULLNAME_=PRIMARY" }, "piute": { - "fips": "49031 - Piute", - "match_fields": "NAME NAME", "compare_fields": "PREDIR PREDIR; NAME NAME; POSTTYPE POSTTYPE; POSTDIR POSTDIR; FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R", - "field_mappings": "FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTTYPE=POSTTYPE_; POSTDIR=POSTDIR_" + "field_mappings": "FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTTYPE=POSTTYPE_; POSTDIR=POSTDIR_", + "fips": "49031 - Piute", + "match_fields": "NAME NAME" }, "rich": { - "fips": "49033 - Rich", - "match_fields": "S_NAME S_NAME", "compare_fields": "PRE_DIR PRE_DIR; S_NAME S_NAME; S_TYPE S_TYPE; SUF_DIR SUF_DIR; ACS_ALIAS ACS_ALIAS; ALIAS1 ALIAS1; ALIAS1_TYP ALIAS1_TYP; ALIAS2 ALIAS2; ALIAS2_TYP ALIAS2_TYP; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PRE_DIR; NAME=S_NAME; LOCAL_UID=CO_UNIQUE; POSTTYPE=S_TYPE; A1_POSTTYPE=ALIAS1_TYP; A2_POSTTYPE=ALIAS2_TYP; ONEWAY=ONE_WAY; DOT_SRFTYP=S_SURF; SPEED_LMT=SPD_LMT", + "fips": "49033 - Rich", + "match_fields": "S_NAME S_NAME", "parse_sources": "ALIAS1=A1; ALIAS2=A2; ACS_ALIAS=A1" }, "saltlake": { - "fips": "49035 - Salt Lake", - "match_fields": "NAME NAME", "compare_fields": "PREDIR PREDIR; NAME NAME; POSTTYPE POSTTYPE; POSTDIR POSTDIR; AN_NAME AN_NAME; AN_POSTDIR AN_POSTDIR; FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R", - "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTDIR=POSTDIR_; AN_NAME=AN_NAME_; POSTTYPE=POSTTYPE_; STATUS=STATUS_; ONEWAY=ONEWAY_; DOT_SRFTYP=DOT_SRFTYP_" + "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTDIR=POSTDIR_; AN_NAME=AN_NAME_; POSTTYPE=POSTTYPE_; STATUS=STATUS_; ONEWAY=ONEWAY_; DOT_SRFTYP=DOT_SRFTYP_", + "fips": "49035 - Salt Lake", + "match_fields": "NAME NAME" }, "sanjuan": { - "fips": "49037 - San Juan", - "match_fields": "S_NAME S_NAME", "compare_fields": "S_NAME S_NAME", "field_mappings": "NAME=S_NAME; LOCAL_UID=CO_UNIQUE; DOT_CLASS=CLASS", + "fips": "49037 - San Juan", + "match_fields": "S_NAME S_NAME", "parse_sources": "S_NAME=PRIMARY" }, "sanpete": { - "fips": "49039 - Sanpete", - "match_fields": "", "compare_fields": "", - "field_mappings": "" + "field_mappings": "??", + "fips": "49039 - Sanpete", + "match_fields": "" }, "sevier": { - "fips": "49041 - Sevier", - "match_fields": "S_NAME S_NAME", "compare_fields": "L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD; PRE_DIR PRE_DIR; S_NAME S_NAME; S_TYPE S_TYPE; SUR_DIR SUR_DIR; ALIAS ALIAS", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PRE_DIR; NAME=STREET; LOCAL_UID=CO_UNIQUE; POSTTYPE=S_TYPE", + "fips": "49041 - Sevier", + "match_fields": "S_NAME S_NAME", "parse_sources": "STREET=PRIMARY; ALIAS=A1" }, "summit": { - "fips": "49043 - Summit", - "match_fields": "STREET STREET", "compare_fields": "PREFIX_DIR PREFIX_DIR; STREET STREET; CLASSIFICA CLASSIFICA; POST_DIR POST_DIR; FROMLEFT FROMLEFT; TOLEFT TOLEFT; FROMRIGHT FROMRIGHT; TORIGHT TORIGHT", - "field_mappings": "FROMADDR_L=FROMLEFT; TOADDR_L=TOLEFT; FROMADDR_R=FROMRIGHT; TOADDR_R=TORIGHT; PREDIR=PREFIX_DIR; NAME=STREET; POSTDIR=POST_DIR; POSTTYPE=SUFF_TYPE; ONEWAY=ONEWAY_; STATUS=STATUS_" + "field_mappings": "FROMADDR_L=FROMLEFT; TOADDR_L=TOLEFT; FROMADDR_R=FROMRIGHT; TOADDR_R=TORIGHT; PREDIR=PREFIX_DIR; NAME=STREET; POSTDIR=POST_DIR; POSTTYPE=SUFF_TYPE; ONEWAY=ONEWAY_; STATUS=STATUS_", + "fips": "49043 - Summit", + "match_fields": "STREET STREET" }, "tooele": { - "fips": "49045 - Tooele", - "match_fields": "FULLNAME FULLNAME", "compare_fields": "FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R; FULLNAME FULLNAME; PREDIR PREDIR; A1_NAME A1_NAME; A2_NAME A2_NAME; AN_NAME AN_NAME; STATUS STATUS", - "uppercase_normalize_fields": ["FULLNAME", "A1_NAME", "A2_NAME"], + "exclude_if_any": "EXCLUDE=X,P; STATUS_=CONSTRUCTION,PLANNED", "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTTYPE=POSTTYPE_; POSTDIR=POSTDIR_; AN_NAME=AN_NAME_; A1_NAME=A1_NAME_; A2_NAME=A2_NAME_; LOCAL_UID=LOCAL_UID_", - "exclude_if_any": "EXCLUDE=X,P; STATUS_=CONSTRUCTION,PLANNED" + "fips": "49045 - Tooele", + "match_fields": "FULLNAME FULLNAME", + "uppercase_normalize_fields": ["FULLNAME", "A1_NAME", "A2_NAME"] }, "uintah": { - "fips": "49047 - Uintah", - "match_fields": "S_NAME S_NAME", "compare_fields": "toleft toleft; toright toright; fromleft fromleft; fromright fromright; S_NAME S_NAME; ALIAS1 ALIAS1", - "uppercase_normalize_fields": ["S_NAME", "ALIAS1"], "field_mappings": "FROMADDR_L=fromleft; TOADDR_L=toleft; FROMADDR_R=fromright; TOADDR_R=toright; PREDIR=PRE_DIR; NAME=S_NAME; POSTDIR=SUF_DIR; A1_NAME=ALIAS1; LOCAL_UID=CO_UNIQUE; POSTTYPE=S_TYPE; SPEED_LMT=SPD_LMT", - "parse_sources": "S_NAME=PRIMARY; ALIAS1=A1" + "fips": "49047 - Uintah", + "match_fields": "S_NAME S_NAME", + "parse_sources": "S_NAME=PRIMARY; ALIAS1=A1", + "uppercase_normalize_fields": ["S_NAME", "ALIAS1"] }, "utah": { + "compare_fields": "ROADPREDIR ROADPREDIR; ROADNAME ROADNAME; ROADTYPE ROADTYPE; ROADPOSTDIR ROADPOSTDIR; ALTROADNAME ALTROADNAME; ALTROADTYPE ALTROADTYPE; ALTROADNAME2 ALTROADNAME2; ALTROADTYPE2 ALTROADTYPE2; FROMLEFT FROMLEFT; TOLEFT TOLEFT; FROMRIGHT FROMRIGHT; TORIGHT TORIGHT", + "custom_handler": "utah_road_names", + "field_mappings": "FROMADDR_L=FROMLEFT; TOADDR_L=TOLEFT; FROMADDR_R=FROMRIGHT; TOADDR_R=TORIGHT; PREDIR=ROADPREDIR; NAME=ROADNAME; POSTTYPE=ROADTYPE; POSTDIR=ROADPOSTDIR; A1_NAME=ALTROADNAME; A1_POSTTYPE=ALTROADTYPE; A2_NAME=ALTROADNAME2", "fips": "49049 - Utah", "match_fields": "ROADNAME ROADNAME", - "compare_fields": "ROADPREDIR ROADPREDIR; ROADNAME ROADNAME; ROADTYPE ROADTYPE; ROADPOSTDIR ROADPOSTDIR; ALTROADNAME ALTROADNAME; ALTROADTYPE ALTROADTYPE; ALTROADNAME2 ALTROADNAME2; ALTROADTYPE2 ALTROADTYPE2; FROMLEFT FROMLEFT; TOLEFT TOLEFT; FROMRIGHT FROMRIGHT; TORIGHT TORIGHT", - "field_mappings": "FROMADDR_L=FROMLEFT; TOADDR_L=TOLEFT; FROMADDR_R=FROMRIGHT; TOADDR_R=TORIGHT; PREDIR=ROADPREDIR; NAME=ROADNAME" + "rules": ["remove_posttype_if_numeric"] }, "wasatch": { + "compare_fields": "FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R; FULLNAME FULLNAME; PREDIR PREDIR; AN_NAME AN_NAME", + "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTTYPE=POSTTYPE_; POSTDIR=POSTDIR_; AN_NAME=AN_NAME_; LOCAL_UID=LOCAL_UID_", "fips": "49051 - Wasatch", "match_fields": "FULLNAME FULLNAME", - "compare_fields": "FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R; FULLNAME FULLNAME; PREDIR PREDIR; AN_NAME AN_NAME", - "uppercase_normalize_fields": ["FULLNAME"], - "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; POSTTYPE=POSTTYPE_; POSTDIR=POSTDIR_; AN_NAME=AN_NAME_; LOCAL_UID=LOCAL_UID_" + "uppercase_normalize_fields": ["FULLNAME"] }, "washington": { - "fips": "49053 - Washington", - "match_fields": "NAME NAME", "compare_fields": "FROMADDR_L FROMADDR_L; TOADDR_L TOADDR_L; FROMADDR_R FROMADDR_R; TOADDR_R TOADDR_R; PREDIR PREDIR; NAME NAME; POSTTYPE POSTTYPE; POSTDIR POSTDIR; SUFFIXDIR SUFFIXDIR; AN_NAME AN_NAME; AN_POSTDIR AN_POSTDIR; A1_NAME A1_NAME; A1_POSTTYPE A1_POSTTYPE; A2_NAME A2_NAME; A2_POSTTYPE A2_POSTTYPE", - "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; AN_NAME=AN_NAME_; A1_NAME=A1_NAME_; A2_NAME=A2_NAME_; LOCAL_UID=LOCAL_UID_" + "field_mappings": "FULLNAME=FULLNAME_; FROMADDR_L=FROMADDR_L_; TOADDR_L=TOADDR_L_; FROMADDR_R=FROMADDR_R_; TOADDR_R=TOADDR_R_; PREDIR=PREDIR_; NAME=NAME_; AN_NAME=AN_NAME_; A1_NAME=A1_NAME_; A2_NAME=A2_NAME_; LOCAL_UID=LOCAL_UID_", + "fips": "49053 - Washington", + "match_fields": "NAME NAME" }, "wayne": { - "fips": "49055 - Wayne", - "match_fields": "S_NAME S_NAME", "compare_fields": "PRE_DIR PRE_DIR; S_NAME S_NAME; S_TYPE S_TYPE; SUF_DIR SUF_DIR; ACS_ALIAS ACS_ALIAS; ALIAS1 ALIAS1; ALIAS1_TYP ALIAS1_TYP; L_F_ADD L_F_ADD; L_T_ADD L_T_ADD; R_F_ADD R_F_ADD; R_T_ADD R_T_ADD", "field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; FROMADDR_R=R_F_ADD; TOADDR_R=R_T_ADD; PREDIR=PRE_DIR; NAME=S_NAME; LOCAL_UID=CO_UNIQUE; DOT_CLASS=CLASS; ONEWAY=ONE_WAY; DOT_SRFTYP=S_SURF2; VERT_LEVEL=VERTLEVEL", + "fips": "49055 - Wayne", + "match_fields": "S_NAME S_NAME", "parse_sources": "S_NAME=PRIMARY; ALIAS1=A1" }, "weber": { - "fips": "49057 - Weber", - "match_fields": "S_NAME S_NAME", "compare_fields": "PREDIR PREDIR; S_NAME S_NAME; STREETTYPE STREETTYPE; SUFDIR SUFDIR; ALIAS ALIAS; ACS_ALIAS ACS_ALIAS; SUFFIX_911 SUFFIX_911; LEFTFROM LEFTFROM; LEFTTO LEFTTO; RIGHTFROM RIGHTFROM; RIGHTTO RIGHTTO", - "field_mappings": "FROMADDR_L=LEFTFROM; TOADDR_L=LEFTTO; FROMADDR_R=RIGHTFROM; TOADDR_R=RIGHTTO; PREDIR=PREDIR; NAME=S_NAME; POSTDIR=SUFDIR; LOCAL_UID=S_UNIQUE" + "field_mappings": "FROMADDR_L=LEFTFROM; TOADDR_L=LEFTTO; FROMADDR_R=RIGHTFROM; TOADDR_R=RIGHTTO; PREDIR=PREDIR; NAME=S_NAME; POSTDIR=SUFDIR; LOCAL_UID=S_UNIQUE", + "fips": "49057 - Weber", + "match_fields": "S_NAME S_NAME" } } diff --git a/cli/tests/test_cli.py b/cli/tests/test_cli.py index 774e682..f4d087b 100644 --- a/cli/tests/test_cli.py +++ b/cli/tests/test_cli.py @@ -78,7 +78,7 @@ def test_county_boundary_name_is_uppercase(county, expected): def test_saltlake_profile_replaces_vecc(): profiles = load_profiles() - assert profiles["saltlake"].require("fips") == "49035" + assert profiles["saltlake"].require("fips") == "49035 - Salt Lake" assert "vecc" not in profiles diff --git a/cli/tests/test_etl_mappers.py b/cli/tests/test_etl_mappers.py new file mode 100644 index 0000000..488deb2 --- /dev/null +++ b/cli/tests/test_etl_mappers.py @@ -0,0 +1,123 @@ +import pytest + +from utrans.etl_handlers import HANDLERS, profile_handler +from utrans.etl_mappers import _validate_value_mappings, _value_mappings +from utrans.etl_rules import apply_rules, profile_rules +from utrans.profiles import CountyProfile + + +def test_value_mappings_normalizes_fields_and_source_values(): + profile = CountyProfile( + key="example", + values={ + "value_mappings": { + "oneway": {" One Direction ": "Y"}, + } + }, + ) + + assert _value_mappings(profile) == {"ONEWAY": {"ONE DIRECTION": "Y"}} + + +@pytest.mark.parametrize( + "value, message", + [ + ([], "must be an object"), + ({"ONEWAY": []}, "map field names to objects"), + ({"ONEWAY": {"ONE DIRECTION": 1}}, "must map strings to strings"), + ], +) +def test_value_mappings_reject_invalid_configuration(value, message): + profile = CountyProfile(key="example", values={"value_mappings": value}) + + with pytest.raises(TypeError, match=message): + _value_mappings(profile) + + +def test_value_mappings_validate_targets_and_domain_values(): + profile = CountyProfile(key="example", values={}) + + with pytest.raises(ValueError, match="unknown target field"): + _validate_value_mappings(profile, {"ONEWAY": {}}, {}, {}) + + with pytest.raises(ValueError, match="outside the target domain"): + _validate_value_mappings( + profile, + {"ONEWAY": {"ONE DIRECTION": "MAYBE"}}, + {"ONEWAY": "ONEWAY"}, + {"ONEWAY": {"Y": "Y", "YES": "Y"}}, + ) + + +def test_rules_apply_legacy_name_cleanup(): + row: list[object] = ["1200", "RD", "N"] + indexes = {"NAME": 0, "POSTTYPE": 1, "POSTDIR": 2} + + apply_rules(row, indexes, ["remove_posttype_if_numeric"]) + + assert row == ["1200", "", "N"] + + +def test_rules_reject_unknown_and_duplicate_names(): + profile = CountyProfile(key="example", values={"rules": ["unknown"]}) + + with pytest.raises(ValueError, match="unsupported"): + profile_rules(profile) + + profile = CountyProfile( + key="example", values={"rules": ["remove_posttype_if_numeric"] * 2} + ) + with pytest.raises(ValueError, match="duplicates"): + profile_rules(profile) + + +def test_profile_handler_uses_closed_registry(): + profile = CountyProfile(key="utah", values={"custom_handler": "utah_road_names"}) + + assert profile_handler(profile) is HANDLERS["utah_road_names"] + + profile = CountyProfile(key="example", values={"custom_handler": "os.system"}) + with pytest.raises(ValueError, match="unsupported custom_handler"): + profile_handler(profile) + + +def test_utah_handler_parses_numeric_primary_and_alias_names(): + row: list[object] = [ + "1200 NORTH", + "300 WEST", + "DR", + "500 SOUTH", + "", + "", + "", + "RD", + "", + "", + "", + "", + ] + indexes = { + "ROADNAME": 0, + "ALTROADNAME": 1, + "ALTROADTYPE": 2, + "ALTROADNAME2": 3, + "NAME": 4, + "POSTDIR": 5, + "A1_NAME": 6, + "A1_POSTTYPE": 7, + "A2_NAME": 8, + "AN_NAME": 9, + "AN_POSTDIR": 10, + "POSTTYPE": 11, + } + + HANDLERS["utah_road_names"].apply( + row, + indexes, + {"A1_POSTTYPE": {"DR": "DR"}}, + ) + + assert row[4:6] == ["1200", "N"] + assert row[6:9] == ["", "", ""] + assert row[9:11] == ["300", "W"] + assert row[11] == "" From 21961c2a3abf418a82d9e0028c49c8ccddfdcc9c Mon Sep 17 00:00:00 2001 From: Scott Davis Date: Mon, 24 Aug 2026 15:59:31 -0600 Subject: [PATCH 10/13] chore: UTrans -> UTRANS --- README.md | 10 +++++----- add-in/README.md | 6 +++--- add-in/UGRC.UtransTools/Config.daml | 18 +++++++++--------- .../UtransEditorDockpaneView.xaml | 4 ++-- cli/pyproject.toml | 2 +- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 254b31e..4056c98 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,18 @@ -# UTrans Tools +# UTRANS Tools -Tools for working with UTrans data +Tools for working with UTRANS data ## CLI -This is a Python CLI tool for processing county-submitted data prior to ingestion into UTrans. See its [README](cli/README.md) for installation, usage, and release instructions. +This is a Python CLI tool for processing county-submitted data prior to ingestion into UTRANS. See its [README](cli/README.md) for installation, usage, and release instructions. ## ArcGIS Pro Add-in -This is an ArcGIS Pro Add-in for working with UTrans data. See its [README](add-in/README.md) for installation, usage, and release instructions. +This is an ArcGIS Pro Add-in for working with UTRANS data. See its [README](add-in/README.md) for installation, usage, and release instructions. ## Data Flow -The following diagram shows the flow of data from the county into UTrans. +The following diagram shows the flow of data from the county into UTRANS. ```mermaid flowchart TD diff --git a/add-in/README.md b/add-in/README.md index 53ce0b8..1b4b468 100644 --- a/add-in/README.md +++ b/add-in/README.md @@ -1,10 +1,10 @@ -# UTrans Tools ArcGIS Pro Add-In +# UTRANS Tools ArcGIS Pro Add-In ## Usage ### Adding New Features -Use **Add New** to copy an unlinked new record or records from `DFC_RESULT` into the UTrans roads layer. +Use **Add New** to copy an unlinked new record or records from `DFC_RESULT` into the UTRANS roads layer. 1. In the map, select one or more unlinked records in the `DFC_RESULT` layer. A record is eligible when its change type is `N` and its `BASE_FID` is `-1`. 2. Review or update the road attributes shown in the editor. When multiple records are selected, the values are used for each new road. @@ -12,7 +12,7 @@ Use **Add New** to copy an unlinked new record or records from `DFC_RESULT` into When multiple features are selected, the form values override the county-road values for `CARTOCODE`, `ONEWAY`, `VERT_LEVEL`, `SPEED_LMT`, and `STATUS`. Review these values before clicking **Add New** because the same editor values are applied to every new road. -The add-in creates a UTrans road using each selected county-road geometry and attributes, links the `DFC_RESULT` record to the new road, and marks the DFC disposition as `COMPLETED`. The map selections are cleared after the operation finishes. +The add-in creates a UTRANS road using each selected county-road geometry and attributes, links the `DFC_RESULT` record to the new road, and marks the DFC disposition as `COMPLETED`. The map selections are cleared after the operation finishes. ## Developer Set Up diff --git a/add-in/UGRC.UtransTools/Config.daml b/add-in/UGRC.UtransTools/Config.daml index 03eea27..3107f22 100644 --- a/add-in/UGRC.UtransTools/Config.daml +++ b/add-in/UGRC.UtransTools/Config.daml @@ -6,7 +6,7 @@ xsi:schemaLocation="http://schemas.esri.com/DADF/Registry file:///C:/Program%20Files/ArcGIS/Pro/bin/ArcGIS.Desktop.Framework.xsd"> - UGRC_UtransTools + UGRC_UtransTools UGRC_UtransTools description Images\AddinDesktop32.png stdavis @@ -18,36 +18,36 @@ - + - + - +