Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"python-envs.defaultEnvManager": "ms-python.python:conda",
"cSpell.words": [
"ACCESSCODE",
"ALTROADNAME",
"ALTROADTYPE",
"arcgispro",
"arcpy",
"boxelder",
Expand All @@ -25,12 +27,18 @@
"pytest",
"pythonpath",
"recents",
"ROADALIASNAME",
"saltlake",
"Sevier",
"SRFTYP",
"STREETNAME",
"SUFDIR",
"SUFFIXDIR",
"testpaths",
"TOADDR",
"Tooele",
"UGRC",
"Uintah",
"utrans",
"UTRANS",
"vecc"
Expand All @@ -42,6 +50,18 @@
"/^dotnet build add-in\\\\UGRC\\.UtransTools\\\\UGRC\\.UtransTools\\.csproj --no-restore$/": {
"approve": true,
"matchCommandLine": true
},
"/^conda activate utrans-tools$/": {
"approve": true,
"matchCommandLine": true
},
"/^conda activate utrans-tools; pytest[^;]*$/": {
"approve": true,
"matchCommandLine": true
}
"/^conda activate utrans-tools; ruff/": {
"approve": true,
"matchCommandLine": true
}
},
"[csharp]": {
Expand Down
10 changes: 5 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
# UTrans Tools
# UTRANS Tools

Tools for working with UTrans data
Tools for working with UTRANS data

## CLI

This is a Python CLI tool for processing county-submitted data prior to ingestion into UTrans. See its [README](cli/README.md) for installation, usage, and release instructions.
This is a Python CLI tool for processing county-submitted data prior to ingestion into UTRANS. See its [README](cli/README.md) for installation, usage, and release instructions.

## ArcGIS Pro Add-in

This is an ArcGIS Pro Add-in for working with UTrans data. See its [README](add-in/README.md) for installation, usage, and release instructions.
This is an ArcGIS Pro Add-in for working with UTRANS data. See its [README](add-in/README.md) for installation, usage, and release instructions.

## Data Flow

The following diagram shows the flow of data from the county into UTrans.
The following diagram shows the flow of data from the county into UTRANS.

```mermaid
flowchart TD
Expand Down
6 changes: 3 additions & 3 deletions add-in/README.md
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
# UTrans Tools ArcGIS Pro Add-In
# UTRANS Tools ArcGIS Pro Add-In

## Usage

### Adding New Features

Use **Add New** to copy an unlinked new record or records from `DFC_RESULT` into the UTrans roads layer.
Use **Add New** to copy an unlinked new record or records from `DFC_RESULT` into the UTRANS roads layer.

1. In the map, select one or more unlinked records in the `DFC_RESULT` layer. A record is eligible when its change type is `N` and its `BASE_FID` is `-1`.
2. Review or update the road attributes shown in the editor. When multiple records are selected, the values are used for each new road.
3. Click **Add New**.

When multiple features are selected, the form values override the county-road values for `CARTOCODE`, `ONEWAY`, `VERT_LEVEL`, `SPEED_LMT`, and `STATUS`. Review these values before clicking **Add New** because the same editor values are applied to every new road.

The add-in creates a UTrans road using each selected county-road geometry and attributes, links the `DFC_RESULT` record to the new road, and marks the DFC disposition as `COMPLETED`. The map selections are cleared after the operation finishes.
The add-in creates a UTRANS road using each selected county-road geometry and attributes, links the `DFC_RESULT` record to the new road, and marks the DFC disposition as `COMPLETED`. The map selections are cleared after the operation finishes.

## Developer Set Up

Expand Down
18 changes: 9 additions & 9 deletions add-in/UGRC.UtransTools/Config.daml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
xsi:schemaLocation="http://schemas.esri.com/DADF/Registry file:///C:/Program%20Files/ArcGIS/Pro/bin/ArcGIS.Desktop.Framework.xsd">

<AddInInfo id="{f893be2f-ad95-46be-9d60-fd6dba457152}" version="1.14" desktopVersion="3.6.59527" product="ArcGISPro">
<Name>UGRC_UtransTools</Name>
<Name>UGRC_UtransTools</Name>
<Description>UGRC_UtransTools description</Description>
<Image>Images\AddinDesktop32.png</Image>
<Author>stdavis</Author>
Expand All @@ -18,36 +18,36 @@
<modules>
<insertModule id="UGRC_UtransTools_Module" className="UtransToolsModule" autoLoad="false" caption="UTRANS Tools">
<tabs>
<tab id="UGRC_UtransTools_CustomTab" caption="UTrans" keytip="UTRANS_Tab">
<tab id="UGRC_UtransTools_CustomTab" caption="UTRANS" keytip="UTRANS_Tab">
<group refID="UGRC_UtransTools_Group1"/>
</tab>
</tabs>

<groups>
<group id="UGRC_UtransTools_Group1" caption="UTrans" keytip="UTRANS_Group">
<group id="UGRC_UtransTools_Group1" caption="UTRANS" keytip="UTRANS_Group">
<button refID="UGRC_UtransTools_UtransEditorDockpaneButton" />
</group>
</groups>

<controls>
<button id="UGRC_UtransTools_UtransEditorDockpaneButton"
caption="UTrans Editor"
caption="UTRANS Editor"
className="UtransEditorDockpaneButton"
loadOnClick="true"
smallImage="Images\AddInDesktop16.png"
largeImage="Images\AddInDesktop32.png"
keytip="UTRANS Editor Pane"/>
</controls>
</controls>

<dockPanes>
<dockPane id="UGRC_UtransTools_UtransEditorDockpane"
caption="UTrans Editor"
caption="UTRANS Editor"
className="UtransEditorDockpaneViewModel"
dock="right">
<content className="UtransEditorDockpaneView" />
</dockPane>
</dockPanes>

</insertModule>
</modules>
</ArcGIS>
4 changes: 2 additions & 2 deletions add-in/UGRC.UtransTools/UtransEditorDockpaneView.xaml
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@
<GroupBox
Grid.Row="3"
Header="{Binding ReviewState.UtransRecordObjectId}"
HeaderStringFormat="Target UTrans Road Segment ({0})"
HeaderStringFormat="Target UTRANS Road Segment ({0})"
Margin="3"
>
<Grid IsEnabled="{Binding HasReviewState}">
Expand Down Expand Up @@ -539,7 +539,7 @@
VerticalAlignment="Bottom"
IsEnabled="{Binding CanEditRoadValues}"
>
<TextBlock VerticalAlignment="Bottom" Text="UTrans Database Version:" />
<TextBlock VerticalAlignment="Bottom" Text="UTRANS Database Version:" />
<TextBlock VerticalAlignment="Bottom" Text="{Binding UtransDatabaseVersion}" />
</StackPanel>

Expand Down
91 changes: 87 additions & 4 deletions cli/PROFILES.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ Each key in the JSON object is a county identifier (e.g. `"grand"`, `"davis"`).

**Type:** `string` | **Required by `etl`**

The county's five-digit FIPS code written to `COUNTY_L` and `COUNTY_R` during ETL.
The county's five-digit FIPS code plus the name written to `COUNTY_L` and `COUNTY_R` during ETL. For example, `"49035 - Salt Lake"`.

### `field_mappings`

Expand All @@ -21,13 +21,95 @@ The county's five-digit FIPS code written to `COUNTY_L` and `COUNTY_R` during ET
A semicolon-delimited list of `UTRANS_FIELD=COUNTY_FIELD` assignments. These map
county source fields into the target schema after colliding source fields have been
renamed with an underscore. When the UTRANS destination has a coded-value domain,
the assignment is validated against that domain from `--utrans-roads`; invalid values
are appended to `UTRANS_NOTES`.
valid aliases are converted to coded values. Invalid values are preserved and
appended to `UTRANS_NOTES` in all cases. An invalid value is also copied to the
destination field when it fits within that field's length; otherwise the destination
field is left at its default value. Values that exceed the destination field length
are reported with the field and value.
Comment thread
stdavis marked this conversation as resolved.

```json
"field_mappings": "FROMADDR_L=L_F_ADD; TOADDR_L=L_T_ADD; NAME=STREETNAME"
```

### `value_mappings`

**Type:** `object` | **Optional** — defaults to `{}`

Per-target mappings for source values that do not match a UTRANS coded-value
domain code or description. Each target field contains an object mapping the
source value to the desired target value. Field names and source values are
matched case-insensitively after surrounding whitespace is removed. The mapped
target value is then resolved against the destination coded-value domain, so it
can be either a coded value or a domain description.

```json
"value_mappings": {
"ONEWAY": {
"ONE DIRECTION": "Y",
"TWO WAY": "N"
},
"DOT_CLASS": {
"LOCAL ROAD": "L"
}
}
```

Mappings run after `field_mappings` selects a source field and before normal
destination-domain matching. If no configured mapping or valid domain match is
found, the original source value is handled as an invalid value: it is appended
to `UTRANS_NOTES` and copied only when it fits the target field length.

### `rules`

**Type:** `string[]` | **Optional** — defaults to `[]`

An ordered list of supported post-mapping transformations. Rules run after field
and full-address mappings, but before the feature is normalized and appended. Unknown
or duplicate names are rejected.

```json
"rules": ["remove_postdir_if_alpha", "remove_posttype_if_numeric"]
```

Supported rules:

- `remove_postdir_if_alpha`: clears `POSTDIR` when `NAME` begins with a letter.
- `remove_posttype_if_numeric`: clears `POSTTYPE` when `NAME` begins with a digit.

### `custom_handler`

**Type:** `string` | **Optional**

Selects a built-in, county-specific transformation for behavior that requires
multiple fields or conditional assignments. Handler names are resolved from the
CLI's fixed registry; profile files cannot name Python modules or functions.
Unknown handler names are rejected before ETL updates begin.

```json
"custom_handler": "utah_road_names"
```

Currently supported handlers:

- `utah_road_names`: preserves Utah County's legacy handling of numeric primary
and alternate road names ending in a cardinal direction.
- `davis_alias`: parses Davis County's `RoadAliasName` into alpha or numeric
aliases while ignoring compound names.
- `washington_postdir_and_aliases`: selects Washington County's primary or
suffix direction and removes aliases duplicated by the numeric alias.
- `weber_alias`: parses Weber County's alpha and numeric alias conventions.
- `summit_names`: prefixes Summit County highway names and conditionally parses
its alternate road name.
- `tooele_numeric_name`: parses numeric Tooele County primary names.
- `emery_compact_aliases`: recognizes Emery County compact numeric aliases such
as `100N`.
- `grand_numeric_acs_name`: retains Grand County's ACS name only when numeric.
- `kane_alias_cleanup`: removes Kane County's invalid `K`-prefixed aliases.
- `sevier_postdir_fallback`: uses Sevier County's suffix direction when address
parsing did not produce one.
- `uintah_exclusion_note`: records Uintah County's excluded features in notes.
- `wayne_acs_alias`: parses Wayne County's ACS alias.

### `parse_sources`

**Type:** `string` | **Optional**
Expand All @@ -40,7 +122,8 @@ names. `TARGET` is `PRIMARY`, `A1`, or `A2`.
**Type:** `boolean` | **Optional** — defaults to `false`

When `true`, legacy vertical values `1`, `2`, and `3` are translated to `0`, `1`,
and `2` before target-domain validation.
and `2` before domain-value matching. Invalid values are still preserved when they
fit within the destination field length and are always appended to `UTRANS_NOTES`.

### `exclude_if_any`

Expand Down
2 changes: 1 addition & 1 deletion cli/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ build-backend = "hatchling.build"
[project]
name = "ugrc-utrans-tools"
version = "1.0.1"
description = "Command-line tools for working with UTrans data."
description = "Command-line tools for working with UTRANS data."
readme = "README.md"
requires-python = ">=3.13"
license = { file = "LICENSE" }
Expand Down
10 changes: 7 additions & 3 deletions cli/src/utrans/etl.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import argparse
import json
import time
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
Expand All @@ -30,8 +31,8 @@ def _default_output_name(county: str) -> str:


def _county_boundary_name(county: str) -> str:
names = {"boxelder": "Box Elder", "saltlake": "Salt Lake", "sanjuan": "San Juan"}
return names.get(county, county.title())
names = {"boxelder": "BOX ELDER", "saltlake": "SALT LAKE", "sanjuan": "SAN JUAN"}
return names.get(county, county.upper())


def _unique_name(prefix: str) -> str:
Expand Down Expand Up @@ -105,7 +106,7 @@ def run_etl(

log(f"Applying {county} county field mappings")
apply_mapper(staging_source, profile, utrans_roads)
normalize_target_fields(staging_source)
normalize_target_fields(staging_source, utrans_roads)

log("Appending transformed roads to the target schema")
arcpy.management.Append(staging_source, staging_output, "NO_TEST")
Expand Down Expand Up @@ -173,6 +174,7 @@ def build_parser(prog: str | None = None) -> argparse.ArgumentParser:


def main(argv: list[str] | None = None, *, prog: str | None = None) -> int:
start_time = time.time()
args = build_parser(prog).parse_args(argv)
Comment thread
stdavis marked this conversation as resolved.
try:
profiles = load_profiles(Path(args.profiles) if args.profiles else None)
Expand All @@ -190,4 +192,6 @@ def main(argv: list[str] | None = None, *, prog: str | None = None) -> int:
except (RuntimeError, TypeError, FileNotFoundError, json.JSONDecodeError) as exc:
log(str(exc))
return 2
elapsed = time.time() - start_time
log(f"Time elapsed: {elapsed:.2f}s")
return 0
Loading