From 72ace5a6434134544c461dd2a5efde54d8a20459 Mon Sep 17 00:00:00 2001 From: mavrick Date: Tue, 17 Nov 2026 22:56:11 -0600 Subject: [PATCH 1/3] test(analysis): compare seeded polygon overlays with GEOS --- .github/workflows/nightly.yml | 84 ++++- .github/workflows/test-rust.yml | 19 ++ crates/analysis/src/tools.rs | 9 +- crates/analysis/tests/reference.rs | 75 +++++ crates/analysis/tests/reference/README.md | 37 +++ crates/analysis/tests/reference/generate.py | 6 +- .../overlay-regressions/clip-mask-seam.json | 6 + .../union-boundary-touch.json | 5 + crates/analysis/tests/reference/overlay.py | 300 ++++++++++++++++++ 9 files changed, 536 insertions(+), 5 deletions(-) create mode 100644 crates/analysis/tests/reference/README.md create mode 100644 crates/analysis/tests/reference/overlay-regressions/clip-mask-seam.json create mode 100644 crates/analysis/tests/reference/overlay-regressions/union-boundary-touch.json create mode 100644 crates/analysis/tests/reference/overlay.py diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b7f1286ad4..24944f4fd5 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2,7 +2,9 @@ name: Nightly # The slow checks pull requests don't wait for: every platform's tests and the release build of # the apps, once a night on main. A night with no new commits since the last successful run is -# skipped. A failure opens an issue (or comments on the open one); the next success closes it. +# skipped for the build. Generated overlays run every night and report minimized disagreements +# separately. A build failure opens an issue (or comments on the open one); the next success +# closes it. # # gh workflow run nightly.yml -R opengeos/RustGIS --ref main @@ -127,6 +129,86 @@ jobs: cargo test --locked -p rustgis-data --test extent_query_properties PROPERTY_CASES=20000 cargo test --locked -p rustgis-data --test date_properties + # Use a fresh seed even when the commit is the same as last night. + overlay: + name: Generated polygon overlays + if: github.repository == 'opengeos/RustGIS' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 20 + outputs: + findings: ${{ steps.findings.outputs.artifact-id }} + env: + CARGO_PROFILE_DEV_DEBUG: line-tables-only + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + - name: Compare overlays and minimize disagreements + id: compare + env: + PROPERTY_CASES: "256" + run: | + set -euo pipefail + cargo test --locked -p rustgis-analysis --test reference --no-run --message-format=json > "$RUNNER_TEMP/reference-build.jsonl" + runner="$(jq -er 'select(.reason == "compiler-artifact" and .target.name == "reference" and .executable != null) | .executable' "$RUNNER_TEMP/reference-build.jsonl")" + PROPERTY_SEED="$(python3 -c 'import secrets; print(secrets.randbits(64))')" + export PROPERTY_SEED + time uv run crates/analysis/tests/reference/generate.py --overlay "$runner" --findings "$RUNNER_TEMP/overlay-findings" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + id: findings + if: failure() && steps.compare.outcome == 'failure' + with: + name: overlay-findings + path: ${{ runner.temp }}/overlay-findings + if-no-files-found: ignore + + overlay-report: + name: Report new overlay disagreements + needs: overlay + # Only the reporting job has issue write access. Manual runs keep findings as artifacts. + if: failure() && github.event_name == 'schedule' && needs.overlay.outputs.findings != '' + runs-on: ubuntu-latest + permissions: + actions: read + issues: write + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: overlay-findings + path: findings + - name: Open issues for new minimized disagreements + env: + GH_TOKEN: ${{ github.token }} + RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + gh api --paginate "repos/$GITHUB_REPOSITORY/issues?state=all&per_page=100" \ + | jq -r '.[] | select(.pull_request | not) | (.body // "") | split("\n")[] | select(startswith("" + if grep -Fxq -- "$marker" reported; then continue; fi + { + echo "The [nightly run]($RUN) on $GITHUB_SHA found a generated $tool disagreement." + echo + jq -r '"Seed `PROPERTY_SEED=\(.seed)`, case index `\(.index)`. \(.detail)"' "$file" + echo + echo 'Save the minimized case below as JSON and pass it to the overlay replay command in crates/analysis/tests/reference/README.md.' + echo + echo '```json' + cat "$file" + echo '```' + echo "$marker" + } > issue.md + gh issue create -R "$GITHUB_REPOSITORY" --title "Generated overlay disagrees with GEOS ($tool, $kind)" --body-file issue.md + echo "$marker" >> reported + done + report: name: Report needs: [changed, rust, workflows] diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 96f4089b4d..60c39a938e 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -82,6 +82,25 @@ jobs: env: PACKAGES: ${{ inputs.packages }} run: cargo nextest run $PACKAGES --locked --profile ci --no-tests=warn + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + if: matrix.os == 'ubuntu-latest' && github.workflow != 'Nightly' && (inputs.packages == '--workspace' || contains(format('{0} ', inputs.packages), '-p rustgis-analysis ')) + - name: Compare generated polygon overlays with GEOS + if: matrix.os == 'ubuntu-latest' && github.workflow != 'Nightly' && (inputs.packages == '--workspace' || contains(format('{0} ', inputs.packages), '-p rustgis-analysis ')) + id: overlay + env: + PACKAGES: ${{ inputs.packages }} + run: | + set -euo pipefail + # Use the test binary already built by nextest with these workspace features. + cargo nextest list $PACKAGES --locked --list-type binaries-only --message-format json > "$RUNNER_TEMP/test-binaries.json" + runner="$(jq -er '."rust-binaries"["rustgis-analysis::reference"]["binary-path"]' "$RUNNER_TEMP/test-binaries.json")" + time uv run crates/analysis/tests/reference/generate.py --overlay "$runner" --findings "$RUNNER_TEMP/overlay-findings" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: failure() && steps.overlay.outcome == 'failure' + with: + name: overlay-findings + path: ${{ runner.temp }}/overlay-findings + if-no-files-found: ignore - name: Flaky tests id: flaky if: ${{ !cancelled() }} diff --git a/crates/analysis/src/tools.rs b/crates/analysis/src/tools.rs index 0bb511315f..bd5e1e9ca3 100644 --- a/crates/analysis/src/tools.rs +++ b/crates/analysis/src/tools.rs @@ -2479,9 +2479,12 @@ fn overlay_on(inp: &dyn Inputs, p: &Json, op: Overlay, threads: Option) - let rest = if inputs.is_empty() { from_multipolygon(&z.poly) } else { - let polys: Vec> = - inputs.iter().filter_map(|r| ds.features.get(*r)?.geometry.as_ref()).filter_map(to_multipolygon).collect(); - from_multipolygon(&z.poly.difference(&union_all(polys.iter().flat_map(|m| m.0.iter())))) + let polygons = + inputs.iter().filter_map(|r| ds.features.get(*r)?.geometry.as_ref()).filter_map(to_multipolygon).flat_map(|m| m.0); + // A redundant union rounds a lone polygon's vertices. That can open a + // passage where it touches the zone boundary, joining separate pieces. + let cover = mask_of(MultiPolygon::new(polygons.collect())); + from_multipolygon(&z.poly.difference(&cover)) }; Ok::<_, String>(rest) }, diff --git a/crates/analysis/tests/reference.rs b/crates/analysis/tests/reference.rs index 76f2068019..8c28bc0016 100644 --- a/crates/analysis/tests/reference.rs +++ b/crates/analysis/tests/reference.rs @@ -347,3 +347,78 @@ fn tools_agree_with_the_reference() { let n = cases["cases"].as_array().map_or(0, Vec::len); assert!(report.is_empty(), "{} of {n} cases differ from the reference:\n{}", report.len(), report.join("\n")); } + +/// The generated oracle uses the exact projected inputs the tool sees. Return each polygon's +/// rings in that plane for Shapely, including the actual output after RustGIS's round trip. +fn overlay_coordinates(g: &Geometry) -> Json { + let Some(polygons) = to_multipolygon(g) else { return serde_json::json!("non-polygon") }; + let rings: Vec<_> = polygons + .0 + .iter() + .map(|p| { + std::iter::once(p.exterior()) + .chain(p.interiors()) + .map(|ring| ring.0.iter().map(|c| [c.x, c.y]).collect::>()) + .collect::>() + }) + .collect(); + serde_json::json!(rings) +} + +/// A batch from generate.py. Test-harness output and tool diagnostics stay off the reply file. +#[test] +#[ignore = "driven by generate.py --overlay with a batch and reply path"] +fn generated_overlay_batch() { + let batch: Json = serde_json::from_reader(std::io::stdin().lock()).expect("overlay batch"); + let mut replies = Vec::new(); + for case in batch.as_array().expect("list of overlay cases") { + let layers = Layers( + ["a", "b"] + .into_iter() + .map(|name| { + let features: Vec<_> = case[name] + .as_array() + .expect("input features") + .iter() + .enumerate() + .map(|(i, g)| serde_json::json!({"type": "Feature", "geometry": g, "properties": {"id": i.to_string()}})) + .collect(); + let collection = serde_json::json!({"type": "FeatureCollection", "features": features}); + let ds = rustgis_data::io::read_geojson(&collection.to_string(), name).expect("generated polygon layer").0; + (name.to_string(), ds) + }) + .collect(), + ); + let projected = |name: &str| { + layers.0[name].features.iter().map(|f| overlay_coordinates(f.geometry.as_ref().expect("generated polygon"))).collect::>() + }; + let mut reply = serde_json::json!({"a": projected("a"), "b": projected("b")}); + for tool in case["tools"].as_array().expect("overlay tools").iter().map(|t| t.as_str().expect("tool id")) { + let mask = match tool { + "clip" | "erase" => tool, + _ => "overlay", + }; + let params = serde_json::json!({"input": "a", (mask): "b"}); + reply[tool] = match rustgis_analysis::run(tool, &layers, ¶ms) { + Ok(result) => { + let ds = layer_of(result.output).expect("overlay layer"); + let keys = if mask == "overlay" { vec!["id".into(), "id_2".into()] } else { vec!["id".into()] }; + let rows: Vec<_> = ds + .features + .iter() + .enumerate() + .map(|(i, f)| { + let key = key_text(&ds, i, Some(&keys)); + serde_json::json!([key, f.geometry.as_ref().map(overlay_coordinates)]) + }) + .collect(); + serde_json::json!(rows) + } + Err(e) => serde_json::json!(e), + }; + } + replies.push(reply); + } + let path = std::env::var_os("RUSTGIS_OVERLAY_REPLY").expect("reply path from generate.py"); + serde_json::to_writer(std::fs::File::create(path).expect("create overlay reply"), &replies).expect("write overlay reply"); +} diff --git a/crates/analysis/tests/reference/README.md b/crates/analysis/tests/reference/README.md new file mode 100644 index 0000000000..0a132b62fb --- /dev/null +++ b/crates/analysis/tests/reference/README.md @@ -0,0 +1,37 @@ +# Generated polygon overlays + +The fixed corpus in this directory checks known results. The generated overlay check also runs +Clip, Intersect, Union and Erase on seeded polygon pairs and layers with several features. It +varies concave outlines, holes, thin overlaps, shared edges, collinear overlaps, nearly touching +corners and multipart features. + +The reference test sends the same Web Mercator input coordinates that RustGIS uses to Shapely's +GEOS. It compares each input ID's output (both IDs for Intersect and Union) by the area of the +symmetric difference. Ring order, orientation and redundant vertices do not affect that area. +Boundary-only GEOS results and empty polygons count as absent. The comparison checks output +validity and uses the fixed corpus's 0.1 ppm tolerance, scaled by the combined input area +because rounding depends on the inputs even when the answer is tiny or empty. It counts +polygon parts above that same area limit. + +From the repository root, build the reference test and pass the executable printed by Cargo: + +```sh +cargo test --locked -p rustgis-analysis --test reference --no-run +uv run crates/analysis/tests/reference/generate.py --overlay "target/debug/deps/reference-" --findings ./overlay-findings +``` + +CI runs six fixed seeds with 16 cases each. To replay a nightly run, set +`PROPERTY_SEED= PROPERTY_CASES=256` before the `uv run` command. To replay just a +minimized case, save the finding from the issue or run artifact as JSON and add +`--case path/to/finding.json`. Use a fresh `--findings` path each time. + +The reducer keeps removing features, parts, holes and vertices until no single deletion +preserves the same kind of difference for the tool on valid inputs. The nightly run keeps these +findings as an artifact and uses the tool, difference kind and normalized reduced input to skip +issues it has already reported. It groups Clip splits that differ only by a gap a few overlay +grid steps wide between masks, with each reduced variant in the artifact. + +The default run also checks the inputs in `overlay-regressions/`. The Union fixture checks +that a polygon touching the zone boundary leaves the correct number of pieces. The Clip +fixture records a known split where individually rounded masks should meet. The default +run permits that specific difference, and `--case` reports it as a failure. diff --git a/crates/analysis/tests/reference/generate.py b/crates/analysis/tests/reference/generate.py index 66cd9c5d06..3bdd8dd217 100644 --- a/crates/analysis/tests/reference/generate.py +++ b/crates/analysis/tests/reference/generate.py @@ -557,6 +557,10 @@ def check(): if __name__ == "__main__": + if sys.argv[1:2] == ["--overlay"]: + import overlay as generated_overlay + + sys.exit(generated_overlay.run(sys.argv[2:])) if sys.argv[1:] not in ([], ["--check"]): - sys.exit(f"usage: uv run generate.py [--check] (not {' '.join(sys.argv[1:])})") + sys.exit(f"usage: uv run generate.py [--check | --overlay RUNNER --findings DIR] (not {' '.join(sys.argv[1:])})") sys.exit(check() if sys.argv[1:] == ["--check"] else main()) diff --git a/crates/analysis/tests/reference/overlay-regressions/clip-mask-seam.json b/crates/analysis/tests/reference/overlay-regressions/clip-mask-seam.json new file mode 100644 index 0000000000..d47881ec71 --- /dev/null +++ b/crates/analysis/tests/reference/overlay-regressions/clip-mask-seam.json @@ -0,0 +1,6 @@ +{ + "seed": "13714318419105429329", + "index": 614, + "known": "gap between rounded clip masks", + "case": {"a":[{"type":"Polygon","coordinates":[[[-25.01953125,42.0],[-25.0,41.9833984375],[-25.0,42.0],[-25.01953125,42.0]]]}],"b":[{"type":"Polygon","coordinates":[[[-25.029296875,41.9931640625],[-25.009765625,41.9931640625],[-25.009765625,42.009765625],[-25.029296875,41.9931640625]]]},{"type":"Polygon","coordinates":[[[-25.009765625,42.009765625],[-25.009765625,41.9736328125],[-24.990234375,42.009765625],[-25.009765625,42.009765625]]]}],"tools":["clip"]} +} diff --git a/crates/analysis/tests/reference/overlay-regressions/union-boundary-touch.json b/crates/analysis/tests/reference/overlay-regressions/union-boundary-touch.json new file mode 100644 index 0000000000..a3d75302f3 --- /dev/null +++ b/crates/analysis/tests/reference/overlay-regressions/union-boundary-touch.json @@ -0,0 +1,5 @@ +{ + "seed": "42", + "index": 999, + "case": {"a":[{"type":"Polygon","coordinates":[[[-16.9609375,-9.0],[-16.951171875,-9.041015625],[-16.9609375,-9.041015625],[-16.9609375,-9.0]]]}],"b":[{"type":"Polygon","coordinates":[[[-16.9296875,-9.029296875],[-16.94140625,-9.046875],[-16.95703125,-9.041015625],[-16.9765625,-9.041015625],[-16.9296875,-9.029296875]]]}],"tools":["union"]} +} diff --git a/crates/analysis/tests/reference/overlay.py b/crates/analysis/tests/reference/overlay.py new file mode 100644 index 0000000000..af8fe3abf7 --- /dev/null +++ b/crates/analysis/tests/reference/overlay.py @@ -0,0 +1,300 @@ +"""Generated polygon overlays for generate.py's pinned GEOS environment.""" + +import argparse +import hashlib +import json +import os +import random +import subprocess +import tempfile +from pathlib import Path + +import shapely +from shapely.affinity import affine_transform +from shapely.geometry import MultiPolygon, Polygon, box, mapping, shape + +TOOLS = ("clip", "intersect", "union", "erase") +FIXED_SEEDS = (1, 2, 3, 4, 5, 6) +REGRESSIONS = Path(__file__).with_name("overlay-regressions") +MASK_SEAM = "gap between rounded clip masks" +# The fixed corpus allows 0.1 ppm of the answer for the overlay's rounding. Scale by input +# area here: the rounding is driven by the inputs even when an intersection is empty or tiny. +AREA_TOLERANCE = 1e-7 + + +def jagged_polygon(rng, x, width, height): + """Two disjoint chains with increasing x form a simple ring, often concave.""" + xs = sorted(rng.sample(range(x + 1, x + width), rng.randint(3, 9))) + bottom = [(i, rng.randrange(height // 3)) for i in xs] + top = [(i, rng.randrange(2 * height // 3, height)) for i in reversed(xs)] + return Polygon([(x, height // 2), *bottom, (x + width, height // 2), *top]) + + +def generated(seed, count): + """Cycle through the interactions on each seed, varying position, sizes and exact vertices.""" + rng = random.Random(seed) + for i in range(count): + w, h = rng.randint(16, 36), rng.randint(16, 36) + x, y = rng.randrange(3, w - 3), rng.randrange(3, h - 3) + a = [box(0, 0, w, h)] + match i % 8: + case 0: # A strip crosses a hole, splitting the intersection. + hole = box(rng.randint(1, x - 1), rng.randint(1, 4), rng.randint(x + 3, w - 1), h - rng.randint(1, 4)) + a = [Polygon(a[0].exterior.coords, [hole.exterior.coords])] + b = [box(x, -2, x + 2, h + 2)] + case 1: # An oblique sliver or a narrow overlap/gap along a shared side. + delta = rng.choice((-1, 1)) * 2 ** -rng.randint(8, 15) + if delta > 0 and x % 2 == 0: + b = [Polygon([(w - x, y), (w - x + delta, y), (w + x + delta, h + 3), (w + x, h + 3)])] + else: + b = [box(w - delta, y, w + x, h + 3)] + case 2: # Corners that touch or nearly touch on either side. + delta = rng.choice((-1, 0, 1)) * 2 ** -rng.randint(5, 8) + b = [box(w + delta, h + delta, w + x, h + y)] + case 3: # Partial, exactly collinear shared edges, with an extra vertex. + b = [Polygon([(w, y), (w + x, y), (w + x, h + y), (w, h + y), (w, h)])] + case 4: # The boundary of the overlapping area also contains shared edges. + b = [box(x, 0, w + x, h)] + case 5: # Several disjoint parts in each feature. + a = [MultiPolygon([a[0], box(w + 2, 0, 2 * w, h)])] + b = [MultiPolygon([box(x, y, w + x, y + 1), box(2 * w + 2, y, 2 * w + x, h)])] + case 6: # Layers with overlapping features and several candidates per feature. + a.append(box(-x, y, w - x, h + y)) + b = [box(x, -y, w + x, h - y), box(-x, -y, w - x, h + y)] + case 7: # Arbitrary non-axis-aligned polygons, independently sampled. + a, b = ([jagged_polygon(rng, offset, w, h)] for offset in (0, x)) + + # Powers of two preserve coincident coordinates after translation. All shapes stay + # away from the antimeridian and Mercator's latitude limit. + scale = 2 ** -rng.randint(5, 10) + sx, sy = (scale * rng.choice((-1, 1)) for _ in range(2)) + matrix = [sx, 0, 0, sy] if rng.randrange(2) else [0, sx, sy, 0] + matrix += [rng.randint(-160, 160), rng.randint(-72, 72)] + yield { + "a": [mapping(affine_transform(g, matrix)) for g in a], + "b": [mapping(affine_transform(g, matrix)) for g in b], + "tools": TOOLS, + } + + +def polygon_parts(g): + """Discard point and line contacts, even in a mixed GEOS GeometryCollection.""" + if g.geom_type == "Polygon": + return [] if g.is_empty else [g] + if g.geom_type in ("GeometryCollection", "MultiPolygon"): + return [p for member in g.geoms for p in polygon_parts(member)] + return [] + + +def answers(tool, a, b): + """Compare each attribute partition, including the uncovered pieces from both Union layers.""" + if tool in ("clip", "erase"): + mask = shapely.union_all(b) + op = shapely.intersection if tool == "clip" else shapely.difference + return {str(i): op(g, mask) for i, g in enumerate(a)} + + rows = {f"{i}|{j}": g.intersection(h) for i, g in enumerate(a) for j, h in enumerate(b)} + if tool == "union": + cover_a, cover_b = shapely.union_all(a), shapely.union_all(b) + rows.update({f"{i}|": g.difference(cover_b) for i, g in enumerate(a)}) + rows.update({f"|{j}": h.difference(cover_a) for j, h in enumerate(b)}) + return rows + + +def clip_mask_seam(parts, mask, count): + """Identify the known Clip split where independently rounded masks open a tiny gap.""" + if len(mask) < 2 or len(parts) <= count: + return False + x0, y0, x1, y1 = shapely.total_bounds(mask) + # Use a few steps of the overlay integer grid, as Clip does to search for nearby masks. + step = max(x1 - x0, y1 - y0) / 2 ** 27 + return len(polygon_parts(shapely.union_all([p.buffer(step) for p in parts]))) == count + + +def compare(tool, reply, a, b): + """Return the first difference per tool. Area ignores ring direction, start and subdivisions.""" + rows = reply[tool] + if isinstance(rows, str): + return "tool-error", rows + + limit = AREA_TOLERANCE * sum(g.area for g in a + b) + want = answers(tool, a, b) + got = {} + for key, coordinates in rows: + if coordinates == "non-polygon": + return "non-polygon", repr(key) + if not coordinates: + continue # The empty polygon may be encoded as no row, null or no rings. + if key in got: + return "duplicate-row", repr(key) + g = from_rings(coordinates) + if not g.is_valid: + reason = shapely.is_valid_reason(g) + return f"invalid {reason.partition('[')[0].lower()}", f"{key}: {reason}" + got[key] = g + + for key in sorted(got.keys() | want.keys()): + expected = MultiPolygon(polygon_parts(want[key])) if key in want else MultiPolygon() + actual = got.get(key, MultiPolygon()) + off = actual.symmetric_difference(expected).area + if off > limit: + return "area", f"{key}: {off:.9g} m² unshared, limit {limit:.9g} m²" + # Count only parts large enough to resolve at the area tolerance. Counting features + # across keys would conflate real splits with overlapping features from different inputs. + parts = [p for p in actual.geoms if p.area > limit] + n, m = len(parts), sum(p.area > limit for p in expected.geoms) + if n != m: + kind = MASK_SEAM if tool == "clip" and clip_mask_seam(parts, b, m) else "parts" + return kind, f"{key}: {n} polygon parts, GEOS {m} (above {limit:.9g} m²)" + return None + + +def from_rings(polygons): + return MultiPolygon([Polygon(rings[0], rings[1:]) for rings in polygons]) + + +class Runner: + """Use the compiled reference test as a batch adapter, without compiling during shrinking.""" + + def __init__(self, executable, reply): + self.executable = executable.resolve() + self.reply = reply + + def check(self, cases): + self.reply.unlink(missing_ok=True) # A stale reply must never look like a successful batch. + completed = subprocess.run( + [self.executable, "generated_overlay_batch", "--exact", "--ignored"], + input=json.dumps(cases), text=True, stdout=subprocess.PIPE, + env={**os.environ, "RUSTGIS_OVERLAY_REPLY": str(self.reply)}, timeout=60, + ) + if completed.returncode: + raise RuntimeError(f"RustGIS reference adapter failed:\n{completed.stdout}") + replies = json.loads(self.reply.read_text()) + return [self.differences(case, reply) for case, reply in zip(cases, replies, strict=True)] + + @staticmethod + def differences(case, reply): + a, b = ([from_rings(g) for g in reply[side]] for side in ("a", "b")) + if any(not g.is_valid or g.is_empty for g in a + b): + return None # A vertex deletion can invalidate a projected polygon. + return {tool: diff for tool in case["tools"] if (diff := compare(tool, reply, a, b))} + + +def reductions(case): + """Every single deletion, leaving both layers nonempty and each exterior a closed ring.""" + for side in ("a", "b"): + layer = case[side] + for i, g in enumerate(layer): + if len(layer) > 1: + yield {**case, side: layer[:i] + layer[i + 1:]} + parts = list(polygon_parts(shape(g))) + for j, p in enumerate(parts): + if len(parts) > 1: + yield replace(case, side, i, parts[:j] + parts[j + 1:]) + rings = [list(r.coords)[:-1] for r in (p.exterior, *p.interiors)] + for k, ring in enumerate(rings): + candidates = [rings[:k] + rings[k + 1:]] if k else [] + if len(ring) > 3: + candidates += [ + rings[:k] + [ring[:v] + ring[v + 1:]] + rings[k + 1:] + for v in range(len(ring)) + ] + for new in candidates: + polygon = Polygon(new[0], new[1:]) + yield replace(case, side, i, parts[:j] + [polygon] + parts[j + 1:]) + + +def replace(case, side, i, parts): + g = parts[0] if len(parts) == 1 else MultiPolygon(parts) + layer = case[side] + return {**case, side: [*layer[:i], mapping(g), *layer[i + 1:]]} + + +def shrink(runner, case, tool, kind): + """Keep the failure class and tool fixed. Restart after each deletion to a local minimum.""" + current = {**case, "tools": [tool]} + while candidates := list(reductions(current)): + results = runner.check(candidates) + for candidate, differences in zip(candidates, results, strict=True): + if differences and differences.get(tool, (None,))[0] == kind: + current = candidate + break + else: + break + return current + + +def fingerprint(case, tool, kind): + """Identify repeated minimized inputs independent of feature, part and vertex order.""" + sides = [ + sorted(shapely.to_wkb(MultiPolygon(polygon_parts(shape(g))).normalize(), hex=True, byte_order=1) for g in case[side]) + for side in ("a", "b") + ] + return hashlib.sha256(json.dumps([tool, kind, sides]).encode()).hexdigest() + + +def check_regressions(runner): + """Check fixed reduced cases with the same oracle, allowing only the recorded Clip split.""" + paths = sorted(REGRESSIONS.glob("*.json")) + fixtures = [json.loads(path.read_text()) for path in paths] + failures = 0 + for path, fixture, differences in zip(paths, fixtures, runner.check([f["case"] for f in fixtures]), strict=True): + if differences is None: + raise ValueError(f"{path.name}: invalid fixed overlay input") + for tool, (kind, detail) in differences.items(): + known = kind == fixture.get("known") + failures += not known + status = "known disagreement" if known else "disagreement" + print(f"{path.name}: {status} in {tool}: {kind}: {detail}", flush=True) + return failures + + +def run(argv): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("runner", type=Path, help="compiled rustgis-analysis reference test") + parser.add_argument("--case", type=Path, help="replay one minimized failure JSON file") + parser.add_argument("--findings", type=Path, required=True, help="new directory for minimized disagreements") + args = parser.parse_args(argv) + if args.findings.exists(): + parser.error("--findings must be a new path") + count = int(os.environ.get("PROPERTY_CASES", "16")) + if count < 1: + parser.error("PROPERTY_CASES must be positive") + seed = os.environ.get("PROPERTY_SEED") + seeds = FIXED_SEEDS if seed is None else (int(seed),) + if args.case: + replay = json.loads(args.case.read_text()) + series = [(replay["seed"], [(replay["index"], replay["case"])])] + else: + series = [(seed, enumerate(generated(seed, count))) for seed in seeds] + + failures = 0 + with tempfile.TemporaryDirectory() as tmp: + runner = Runner(args.runner, Path(tmp) / "reply.json") + if not args.case and seed is None: + failures += check_regressions(runner) + for seed, indexed in series: + cases = list(indexed) + print(f"Overlay seed {seed}: {len(cases)} cases", flush=True) + results = runner.check([case for _, case in cases]) + for (index, case), differences in zip(cases, results, strict=True): + if differences is None: + raise ValueError(f"seed {seed}, case {index}: invalid generated input") + for tool, (kind, detail) in differences.items(): + failures += 1 + small = shrink(runner, case, tool, kind) + minimized = runner.check([small])[0][tool][1] + digest = fingerprint(small, tool, kind) + args.findings.mkdir(parents=True, exist_ok=True) + file = args.findings / f"{tool}-{digest[:16]}.json" + finding = { + # Keep all 64 bits when tools such as jq read the issue data. + "seed": str(seed), "index": index, "kind": kind, "detail": minimized, + "fingerprint": "clip-mask-seam" if kind == MASK_SEAM else digest, + "case": small, + } + if not file.exists(): + file.write_text(json.dumps(finding, separators=(",", ":")) + "\n") + print(f"{tool} seed {seed} case {index}: {kind}: {detail}; minimized at {file}", flush=True) + print(f"{failures} overlay disagreements") + return int(failures != 0) From e8347f7e106e7836da56ea6dad864232cf47555b Mon Sep 17 00:00:00 2001 From: mavrick Date: Tue, 17 Nov 2026 23:28:46 -0600 Subject: [PATCH 2/3] test(analysis): fail stale known overlay cases; trust only the workflow's issue markers - A fixed overlay case marked known now fails once that disagreement is gone, as reference.rs does for its known cases. - The nightly reporter reads markers only from issues it opened, re-reports the grouped Clip seam once its issue is closed, and opens an issue when the overlay check fails without a minimized finding. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 45 ++++++++++++++++++---- crates/analysis/tests/reference/README.md | 6 ++- crates/analysis/tests/reference/overlay.py | 4 ++ 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 24944f4fd5..bab587ccc8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2,8 +2,9 @@ name: Nightly # The slow checks pull requests don't wait for: every platform's tests and the release build of # the apps, once a night on main. A night with no new commits since the last successful run is -# skipped for the build. Generated overlays run every night and report minimized disagreements -# separately. A build failure opens an issue (or comments on the open one); the next success +# skipped for the build. Generated overlays run every night with a fresh seed: each new minimized +# disagreement opens its own issue, and an overlay run that fails without one is reported like a +# build failure. A build failure opens an issue (or comments on the open one); the next success # closes it. # # gh workflow run nightly.yml -R opengeos/RustGIS --ref main @@ -166,33 +167,40 @@ jobs: if-no-files-found: ignore overlay-report: - name: Report new overlay disagreements + name: Report generated overlays needs: overlay # Only the reporting job has issue write access. Manual runs keep findings as artifacts. - if: failure() && github.event_name == 'schedule' && needs.overlay.outputs.findings != '' + if: ${{ !cancelled() && github.event_name == 'schedule' && (needs.overlay.result == 'success' || needs.overlay.result == 'failure') }} runs-on: ubuntu-latest permissions: actions: read issues: write steps: - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + if: needs.overlay.outputs.findings != '' with: name: overlay-findings path: findings - name: Open issues for new minimized disagreements + if: needs.overlay.outputs.findings != '' env: GH_TOKEN: ${{ github.token }} RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | set -euo pipefail + # Trust only markers in this workflow's own issues, as " " lines. gh api --paginate "repos/$GITHUB_REPOSITORY/issues?state=all&per_page=100" \ - | jq -r '.[] | select(.pull_request | not) | (.body // "") | split("\n")[] | select(startswith("" - if grep -Fxq -- "$marker" reported; then continue; fi + # A reduced input is reported once. The grouped Clip seam class is reported again + # once its issue is closed, so a fix that doesn't hold shows up. + if grep -Fxq -- "open $marker" reported; then continue; fi + if [ "$fingerprint" != clip-mask-seam ] && grep -Fxq -- "closed $marker" reported; then continue; fi { echo "The [nightly run]($RUN) on $GITHUB_SHA found a generated $tool disagreement." echo @@ -206,8 +214,31 @@ jobs: echo "$marker" } > issue.md gh issue create -R "$GITHUB_REPOSITORY" --title "Generated overlay disagrees with GEOS ($tool, $kind)" --body-file issue.md - echo "$marker" >> reported + echo "open $marker" >> reported done + - name: Report a check that failed without findings + env: + GH_TOKEN: ${{ github.token }} + RESULT: ${{ needs.overlay.result }} + FINDINGS: ${{ needs.overlay.outputs.findings }} + RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + title="Nightly generated overlay check failed" + open="$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues?state=open&per_page=100" \ + | jq -sr --arg t "$title" '[.[][] | select(.pull_request | not) | select(.title == $t)][0].number // ""')" + if [ "$RESULT" = success ]; then + if [ -n "$open" ]; then + gh issue close "$open" -R "$GITHUB_REPOSITORY" --comment "Fixed: the nightly overlay check on $GITHUB_SHA passed ($RUN)." + fi + elif [ -z "$FINDINGS" ]; then + if [ -n "$open" ]; then + gh issue comment "$open" -R "$GITHUB_REPOSITORY" --body "Still failing on $GITHUB_SHA: $RUN" + else + gh issue create -R "$GITHUB_REPOSITORY" --title "$title" \ + --body "The generated overlay check on $GITHUB_SHA failed without a minimized finding, so the comparison itself broke: $RUN. This issue closes itself when the check passes." + fi + fi report: name: Report diff --git a/crates/analysis/tests/reference/README.md b/crates/analysis/tests/reference/README.md index 0a132b62fb..a6ab9c3cf1 100644 --- a/crates/analysis/tests/reference/README.md +++ b/crates/analysis/tests/reference/README.md @@ -28,8 +28,10 @@ minimized case, save the finding from the issue or run artifact as JSON and add The reducer keeps removing features, parts, holes and vertices until no single deletion preserves the same kind of difference for the tool on valid inputs. The nightly run keeps these findings as an artifact and uses the tool, difference kind and normalized reduced input to skip -issues it has already reported. It groups Clip splits that differ only by a gap a few overlay -grid steps wide between masks, with each reduced variant in the artifact. +issues it has already reported; it only trusts markers in issues the workflow opened itself. It +groups Clip splits that differ only by a gap a few overlay grid steps wide between masks, with each +reduced variant in the artifact. That group is reported again once its issue is closed. An +overlay run that fails without a finding opens its own issue, which closes when the check passes. The default run also checks the inputs in `overlay-regressions/`. The Union fixture checks that a polygon touching the zone boundary leaves the correct number of pieces. The Clip diff --git a/crates/analysis/tests/reference/overlay.py b/crates/analysis/tests/reference/overlay.py index af8fe3abf7..c32ebedc2f 100644 --- a/crates/analysis/tests/reference/overlay.py +++ b/crates/analysis/tests/reference/overlay.py @@ -246,6 +246,10 @@ def check_regressions(runner): failures += not known status = "known disagreement" if known else "disagreement" print(f"{path.name}: {status} in {tool}: {kind}: {detail}", flush=True) + # As in reference.rs, a known case that agrees again fails until its mark is removed. + if fixture.get("known") and not any(kind == fixture["known"] for kind, _ in differences.values()): + failures += 1 + print(f"{path.name}: the known disagreement is gone; remove the fixture's known mark", flush=True) return failures From 045b45ae2afe649c1a5bde057d72e96026c2c350 Mon Sep 17 00:00:00 2001 From: mavrick Date: Tue, 17 Nov 2026 23:56:01 -0600 Subject: [PATCH 3/3] ci: run the overlay comparison last; report only the workflow's own failure issue - In test-rust.yml the generated overlay steps move to the end of the job, so a disagreement no longer skips the doctests, layering check or release build. - The nightly overlay report finds its failure issue only among issues the workflow opened, and treats a timed-out overlay job as a failure. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 5 +++-- .github/workflows/test-rust.yml | 39 +++++++++++++++++---------------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index bab587ccc8..33a89c66c4 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -170,7 +170,8 @@ jobs: name: Report generated overlays needs: overlay # Only the reporting job has issue write access. Manual runs keep findings as artifacts. - if: ${{ !cancelled() && github.event_name == 'schedule' && (needs.overlay.result == 'success' || needs.overlay.result == 'failure') }} + # A job that ran out of time ends cancelled, but the run isn't: that is a failure too. + if: ${{ !cancelled() && github.event_name == 'schedule' && needs.overlay.result != 'skipped' }} runs-on: ubuntu-latest permissions: actions: read @@ -226,7 +227,7 @@ jobs: set -euo pipefail title="Nightly generated overlay check failed" open="$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues?state=open&per_page=100" \ - | jq -sr --arg t "$title" '[.[][] | select(.pull_request | not) | select(.title == $t)][0].number // ""')" + | jq -sr --arg t "$title" '[.[][] | select(.pull_request | not) | select(.user.login == "github-actions[bot]") | select(.title == $t)][0].number // ""')" if [ "$RESULT" = success ]; then if [ -n "$open" ]; then gh issue close "$open" -R "$GITHUB_REPOSITORY" --comment "Fixed: the nightly overlay check on $GITHUB_SHA passed ($RUN)." diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 60c39a938e..5fa25e9764 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -82,25 +82,6 @@ jobs: env: PACKAGES: ${{ inputs.packages }} run: cargo nextest run $PACKAGES --locked --profile ci --no-tests=warn - - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 - if: matrix.os == 'ubuntu-latest' && github.workflow != 'Nightly' && (inputs.packages == '--workspace' || contains(format('{0} ', inputs.packages), '-p rustgis-analysis ')) - - name: Compare generated polygon overlays with GEOS - if: matrix.os == 'ubuntu-latest' && github.workflow != 'Nightly' && (inputs.packages == '--workspace' || contains(format('{0} ', inputs.packages), '-p rustgis-analysis ')) - id: overlay - env: - PACKAGES: ${{ inputs.packages }} - run: | - set -euo pipefail - # Use the test binary already built by nextest with these workspace features. - cargo nextest list $PACKAGES --locked --list-type binaries-only --message-format json > "$RUNNER_TEMP/test-binaries.json" - runner="$(jq -er '."rust-binaries"["rustgis-analysis::reference"]["binary-path"]' "$RUNNER_TEMP/test-binaries.json")" - time uv run crates/analysis/tests/reference/generate.py --overlay "$runner" --findings "$RUNNER_TEMP/overlay-findings" - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 - if: failure() && steps.overlay.outcome == 'failure' - with: - name: overlay-findings - path: ${{ runner.temp }}/overlay-findings - if-no-files-found: ignore - name: Flaky tests id: flaky if: ${{ !cancelled() }} @@ -145,3 +126,23 @@ jobs: - name: Build release apps if: inputs.release-build run: cargo build --release --locked -p rustgis -p rustgis-cli + # Last, so a disagreement here doesn't skip the doctests, layering check or release build. + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + if: matrix.os == 'ubuntu-latest' && github.workflow != 'Nightly' && (inputs.packages == '--workspace' || contains(format('{0} ', inputs.packages), '-p rustgis-analysis ')) + - name: Compare generated polygon overlays with GEOS + if: matrix.os == 'ubuntu-latest' && github.workflow != 'Nightly' && (inputs.packages == '--workspace' || contains(format('{0} ', inputs.packages), '-p rustgis-analysis ')) + id: overlay + env: + PACKAGES: ${{ inputs.packages }} + run: | + set -euo pipefail + # Use the test binary already built by nextest with these workspace features. + cargo nextest list $PACKAGES --locked --list-type binaries-only --message-format json > "$RUNNER_TEMP/test-binaries.json" + runner="$(jq -er '."rust-binaries"["rustgis-analysis::reference"]["binary-path"]' "$RUNNER_TEMP/test-binaries.json")" + time uv run crates/analysis/tests/reference/generate.py --overlay "$runner" --findings "$RUNNER_TEMP/overlay-findings" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: failure() && steps.overlay.outcome == 'failure' + with: + name: overlay-findings + path: ${{ runner.temp }}/overlay-findings + if-no-files-found: ignore