diff --git a/docs/reference/iox-legacy.md b/docs/reference/iox-legacy.md index fcaaf51b..23820bb6 100644 --- a/docs/reference/iox-legacy.md +++ b/docs/reference/iox-legacy.md @@ -323,6 +323,33 @@ getrennte Vertragsfelder. [#296][v296], [PR #112][i112], unter `-Xmx2g` erfordert Messungen für die zugelassenen Workloads. [Entwurf zu Ressourcen und Geometrie][resource-contract]. +### Lokale Geometrie-Baseline + +Zwölf synthetische [Profile](../../testdata/reference/iox/geometry-cases.json) +liefen am 6. November 2026 mit ilivalidator 1.15.0, Temurin 25.0.2+10 +und `-Xmx2g` auf macOS arm64. Sieben Transfers wurden akzeptiert, +fünf mit den erwarteten Diagnosen abgewiesen. AREA und `areAreas(ALL, …)` +akzeptierten den gemeinsamen geraden Rand und erkannten die Kreuzung. +Bei identischen XTF-2.4-Bytes akzeptierte AREA eine Bogenüberlappung von +rund 0,0481 m bei modellierten 0,050 m. `areAreas(ALL, …)` wies sie mit +seiner Nulltoleranz ab. Die normative Funktionspolicy bleibt offen. +Die separate Rundungsprobe vergleicht gemeinsame Kante und geradlinige +Überlagerung. Beim selbstschneidenden Ring belegt das Log den Ausfall +der AREA-Gruppenprüfung nach dem SURFACE-Fehler. [#468][v468] + +Die zwölf Läufe erreichten bis 76,8 MiB abgetasteten Heap, 160,4 MiB RSS +und 1 ms kumulierte GC-Zeit. Die abgetastete Temp-Spitze stieg in beiden +Gittern von rund 57 KiB bei 100 Polygonen auf 574 KiB bei 1’000 Polygonen. +Beim JVM-Exit war das Temp-Verzeichnis in allen zwölf Läufen leer. +Je drei Wiederholungen der beiden 1’000er-Gitter ergaben Medianwerte von +328 ms für disjunkte Flächen und 316 ms für gemeinsame Ränder. +Nach einer am Ende des Messblocks beobachteten Lastspitze wurde der +letzte Lauf mit gemeinsamen Rändern gezielt wiederholt. Die Zeiten +enthalten den JVM-Start. Die Stichproben können +kurze Heap- und Temp-Spitzen verfehlen. Produktionsdaten und +Kandidatenzahlen blieben offen. +[Benchmarkmodus](../../tools/iox-reference-harness/README.md#geometrie-und-ressourcen-messen) + ## Normbelege | Aussage | Gepinnte PDF-Seiten und Auszüge | diff --git a/testdata/reference/iox/geometry-cases.json b/testdata/reference/iox/geometry-cases.json new file mode 100644 index 00000000..9e0c4f59 --- /dev/null +++ b/testdata/reference/iox/geometry-cases.json @@ -0,0 +1,365 @@ +[ + { + "id": "area-grid-disjoint-100", + "mode": "validate", + "model": "ili23/Reference23.ili", + "xtf": "generated/area-grid-disjoint-100.xtf", + "expectation": "positive", + "workload": { + "profile": "grid-disjoint", + "check": "AREA", + "objects": 100, + "polygons": 100, + "rings": 100, + "straightSegments": 400, + "arcSegments": 0, + "spatialCandidates": null, + "seed": 3, + "gridSpacing": 3 + }, + "requiredInfo": [ + "validate AREA Reference23.Records.Parcel.shape...", + "object count 100 (structured elements 0)" + ], + "diagnostics": [], + "generate": { + "class": "Reference23.Records.Parcel", + "seed": 3, + "spacing": 3 + } + }, + { + "id": "area-grid-disjoint-1000", + "mode": "validate", + "model": "ili23/Reference23.ili", + "xtf": "generated/area-grid-disjoint-1000.xtf", + "expectation": "positive", + "workload": { + "profile": "grid-disjoint", + "check": "AREA", + "objects": 1000, + "polygons": 1000, + "rings": 1000, + "straightSegments": 4000, + "arcSegments": 0, + "spatialCandidates": null, + "seed": 3, + "gridSpacing": 3 + }, + "requiredInfo": [ + "validate AREA Reference23.Records.Parcel.shape...", + "object count 1000 (structured elements 0)" + ], + "diagnostics": [], + "generate": { + "class": "Reference23.Records.Parcel", + "seed": 3, + "spacing": 3 + } + }, + { + "id": "area-grid-shared-edges-100", + "mode": "validate", + "model": "ili23/Reference23.ili", + "xtf": "generated/area-grid-shared-edges-100.xtf", + "expectation": "positive", + "workload": { + "profile": "grid-shared-edges", + "check": "AREA", + "objects": 100, + "polygons": 100, + "rings": 100, + "straightSegments": 400, + "arcSegments": 0, + "spatialCandidates": null, + "seed": 3, + "gridSpacing": 2 + }, + "requiredInfo": [ + "validate AREA Reference23.Records.Parcel.shape...", + "object count 100 (structured elements 0)" + ], + "diagnostics": [], + "generate": { + "class": "Reference23.Records.Parcel", + "seed": 3, + "spacing": 2 + } + }, + { + "id": "area-grid-shared-edges-1000", + "mode": "validate", + "model": "ili23/Reference23.ili", + "xtf": "generated/area-grid-shared-edges-1000.xtf", + "expectation": "positive", + "workload": { + "profile": "grid-shared-edges", + "check": "AREA", + "objects": 1000, + "polygons": 1000, + "rings": 1000, + "straightSegments": 4000, + "arcSegments": 0, + "spatialCandidates": null, + "seed": 3, + "gridSpacing": 2 + }, + "requiredInfo": [ + "validate AREA Reference23.Records.Parcel.shape...", + "object count 1000 (structured elements 0)" + ], + "diagnostics": [], + "generate": { + "class": "Reference23.Records.Parcel", + "seed": 3, + "spacing": 2 + } + }, + { + "id": "area-rounding-shared", + "mode": "validate", + "model": "ili23/Reference23.ili", + "xtf": "generated/area-rounding-shared.xtf", + "expectation": "positive", + "workload": { + "profile": "decimal-rounding", + "check": "AREA", + "objects": 2, + "polygons": 2, + "rings": 2, + "straightSegments": 8, + "arcSegments": 0, + "spatialCandidates": null + }, + "requiredInfo": [ + "validate AREA Reference23.Records.Parcel.shape...", + "object count 2 (structured elements 0)" + ], + "diagnostics": [], + "generate": { + "class": "Reference23.Records.Parcel", + "rings": [ + [["10.00", "10.00"], ["20.00", "10.00"], ["20.00", "20.00"], ["10.00", "20.00"], ["10.00", "10.00"]], + [["19.995", "10.00"], ["30.00", "10.00"], ["30.00", "20.00"], ["19.995", "20.00"], ["19.995", "10.00"]] + ] + } + }, + { + "id": "area-rounding-overlap", + "mode": "validate", + "model": "ili23/Reference23.ili", + "xtf": "generated/area-rounding-overlap.xtf", + "expectation": "negative", + "workload": { + "profile": "decimal-rounding", + "check": "AREA", + "objects": 2, + "polygons": 2, + "rings": 2, + "straightSegments": 8, + "arcSegments": 0, + "spatialCandidates": null + }, + "requiredInfo": [ + "validate AREA Reference23.Records.Parcel.shape...", + "object count 2 (structured elements 0)" + ], + "diagnostics": [ + "Error: Reference23.Records.Parcel: tid parcel-1: Overlay coord1 (19.990, 10.000), coord2 (20.000, 10.000), tids parcel-1, parcel-2", + "Error: Reference23.Records.Parcel: tid parcel-1: Intersection coord1 (19.990, 10.000), tids parcel-1, parcel-2", + "Error: Reference23.Records.Parcel: tid parcel-1: Intersection coord1 (20.000, 10.000), tids parcel-1, parcel-2", + "Error: Reference23.Records.Parcel: tid parcel-1: Intersection coord1 (20.000, 20.000), tids parcel-1, parcel-2", + "Error: Reference23.Records.Parcel: tid parcel-1: Intersection coord1 (19.990, 20.000), tids parcel-1, parcel-2", + "Error: Reference23.Records.Parcel: tid parcel-1: Overlay coord1 (19.990, 20.000), coord2 (20.000, 20.000), tids parcel-1, parcel-2", + "Error: failed to validate AREA Reference23.Records.Parcel.shape" + ], + "generate": { + "class": "Reference23.Records.Parcel", + "rings": [ + [["10.00", "10.00"], ["20.00", "10.00"], ["20.00", "20.00"], ["10.00", "20.00"], ["10.00", "10.00"]], + [["19.994", "10.00"], ["30.00", "10.00"], ["30.00", "20.00"], ["19.994", "20.00"], ["19.994", "10.00"]] + ] + } + }, + { + "id": "area-self-crossing-ring", + "mode": "validate", + "model": "ili23/Reference23.ili", + "xtf": "generated/area-self-crossing-ring.xtf", + "expectation": "negative", + "workload": { + "profile": "invalid-ring", + "check": "SURFACE topology; AREA skipped", + "objects": 1, + "polygons": 1, + "rings": 1, + "straightSegments": 4, + "arcSegments": 0, + "spatialCandidates": null + }, + "requiredInfo": [ + "AREA topology of attribute shape not validated, validation of SURFACE topology failed in attribute shape", + "object count 1 (structured elements 0)" + ], + "diagnostics": [ + "Error: Reference23.Records.Parcel.shape: Intersection coord1 (15.000, 15.000), tids parcel-1, parcel-1" + ], + "forbiddenInfo": [ + "validate AREA Reference23.Records.Parcel.shape..." + ], + "generate": { + "class": "Reference23.Records.Parcel", + "rings": [ + [["10.00", "10.00"], ["20.00", "20.00"], ["20.00", "10.00"], ["10.00", "20.00"], ["10.00", "10.00"]] + ] + } + }, + { + "id": "areareas-shared-edge-23", + "mode": "validate", + "model": "ili23/Geometry23.ili", + "xtf": "generated/areareas-shared-edge-23.xtf", + "expectation": "positive", + "workload": { + "profile": "areAreas-straight", + "check": "areAreas(ALL, ...)", + "objects": 2, + "polygons": 2, + "rings": 2, + "straightSegments": 8, + "arcSegments": 0, + "spatialCandidates": null + }, + "requiredInfo": [ + "validate set constraint Geometry23.Samples.SurfaceParcel.Constraint1...", + "object count 2 (structured elements 0)" + ], + "requiredPools": ["ItfAreaPolygon2Linetable"], + "diagnostics": [], + "forbiddenInfo": [ + "validate AREA Geometry23.Samples.SurfaceParcel.shape..." + ], + "generate": { + "class": "Geometry23.Samples.SurfaceParcel", + "rings": [ + [["10.00", "10.00"], ["20.00", "10.00"], ["20.00", "20.00"], ["10.00", "20.00"], ["10.00", "10.00"]], + [["20.00", "10.00"], ["30.00", "10.00"], ["30.00", "20.00"], ["20.00", "20.00"], ["20.00", "10.00"]] + ] + } + }, + { + "id": "areareas-overlap-23", + "mode": "validate", + "model": "ili23/Geometry23.ili", + "xtf": "generated/areareas-overlap-23.xtf", + "expectation": "negative", + "workload": { + "profile": "areAreas-straight", + "check": "areAreas(ALL, ...)", + "objects": 2, + "polygons": 2, + "rings": 2, + "straightSegments": 8, + "arcSegments": 0, + "spatialCandidates": null + }, + "requiredInfo": [ + "validate set constraint Geometry23.Samples.SurfaceParcel.Constraint1...", + "evaluateAreArea1(): Geometry23.Samples.SurfaceParcel:INTERLIS.areAreas returned false (Validator.java:2658)", + "object count 2 (structured elements 0)" + ], + "requiredPools": ["ItfAreaPolygon2Linetable"], + "diagnostics": [ + "Error: Geometry23.Samples.SurfaceParcel: tid parcel-2/shape[1]: Intersection coord1 (20.000, 15.000), tids parcel-1/shape[1], parcel-2/shape[1]", + "Error: Geometry23.Samples.SurfaceParcel: tid parcel-2/shape[1]: Intersection coord1 (15.000, 20.000), tids parcel-1/shape[1], parcel-2/shape[1]", + "Error: Set Constraint Geometry23.Samples.SurfaceParcel.Constraint1 is not true." + ], + "forbiddenInfo": [ + "validate AREA Geometry23.Samples.SurfaceParcel.shape..." + ], + "generate": { + "class": "Geometry23.Samples.SurfaceParcel", + "rings": [ + [["10.00", "10.00"], ["20.00", "10.00"], ["20.00", "20.00"], ["10.00", "20.00"], ["10.00", "10.00"]], + [["15.00", "15.00"], ["25.00", "15.00"], ["25.00", "25.00"], ["15.00", "25.00"], ["15.00", "15.00"]] + ] + } + }, + { + "id": "area-arc-below-tolerance-24", + "mode": "validate", + "model": "ili24/GeometryArc24.ili", + "xtf": "ili24/arc-below-tolerance.xtf", + "expectation": "positive", + "workload": { + "profile": "arc-tolerance", + "check": "AREA", + "objects": 2, + "polygons": 2, + "rings": 2, + "straightSegments": 7, + "arcSegments": 1, + "spatialCandidates": null + }, + "requiredInfo": [ + "validate AREA GeometryArc24.Records.Parcel.shape...", + "object count 2 (structured elements 0)" + ], + "diagnostics": [] + }, + { + "id": "area-arc-above-tolerance-24", + "mode": "validate", + "model": "ili24/GeometryArc24.ili", + "xtf": "ili24/arc-above-tolerance.xtf", + "expectation": "negative", + "workload": { + "profile": "arc-tolerance", + "check": "AREA", + "objects": 2, + "polygons": 2, + "rings": 2, + "straightSegments": 7, + "arcSegments": 1, + "spatialCandidates": null + }, + "requiredInfo": [ + "validate AREA GeometryArc24.Records.Parcel.shape...", + "object count 2 (structured elements 0)" + ], + "diagnostics": [ + "Error: GeometryArc24.Records.Parcel: tid arc-left: Intersection overlap 0.052109638456386165, coord1 (20.000, 10.000), coord2 (20.000, 29.123), tids arc-left, arc-right", + "Error: failed to validate AREA GeometryArc24.Records.Parcel.shape" + ] + }, + { + "id": "areareas-arc-below-tolerance-24", + "mode": "validate", + "model": "ili24/areareas/GeometryArc24.ili", + "xtf": "ili24/arc-below-tolerance.xtf", + "expectation": "negative", + "workload": { + "profile": "arc-tolerance-policy", + "check": "areAreas(ALL, ...)", + "objects": 2, + "polygons": 2, + "rings": 2, + "straightSegments": 7, + "arcSegments": 1, + "spatialCandidates": null, + "assessment": "UNRESOLVED" + }, + "requiredInfo": [ + "validate set constraint GeometryArc24.Records.Parcel.Constraint1...", + "evaluateAreArea1(): GeometryArc24.Records.Parcel:INTERLIS.areAreas returned false (Validator.java:2658)", + "object count 2 (structured elements 0)" + ], + "diagnostics": [ + "Error: GeometryArc24.Records.Parcel: tid arc-left/shape[1]: Intersection overlap 0.04811791461327175, coord1 (20.000, 10.000), coord2 (20.000, 29.057), tids arc-left/shape[1], arc-right/shape[1]", + "Error: Set Constraint GeometryArc24.Records.Parcel.Constraint1 is not true." + ], + "forbiddenInfo": [ + "validate AREA GeometryArc24.Records.Parcel.shape..." + ] + } +] diff --git a/testdata/reference/iox/ili23/Geometry23.ili b/testdata/reference/iox/ili23/Geometry23.ili new file mode 100644 index 00000000..fad69f1b --- /dev/null +++ b/testdata/reference/iox/ili23/Geometry23.ili @@ -0,0 +1,12 @@ +INTERLIS 2.3; + +MODEL Geometry23 (en) AT "https://example.invalid/iox-reference" VERSION "2026-11-06" = + TOPIC Samples = + DOMAIN Point = COORD 0.00 .. 100.00 [INTERLIS.m], 0.00 .. 100.00 [INTERLIS.m]; + + CLASS SurfaceParcel = + shape : MANDATORY SURFACE WITH (STRAIGHTS, ARCS) VERTEX Point WITHOUT OVERLAPS > 0.01; + SET CONSTRAINT INTERLIS.areAreas(ALL, UNDEFINED, >>shape); + END SurfaceParcel; + END Samples; +END Geometry23. diff --git a/testdata/reference/iox/ili24/GeometryArc24.ili b/testdata/reference/iox/ili24/GeometryArc24.ili new file mode 100644 index 00000000..b18c7f8b --- /dev/null +++ b/testdata/reference/iox/ili24/GeometryArc24.ili @@ -0,0 +1,10 @@ +INTERLIS 2.4; + +MODEL GeometryArc24 (en) AT "https://example.invalid/iox-reference" VERSION "2026-11-06" = + TOPIC Records = + DOMAIN Point = COORD 0.000 .. 100.000 [INTERLIS.m], 0.000 .. 100.000 [INTERLIS.m]; + CLASS Parcel = + shape : MANDATORY AREA WITH (STRAIGHTS, ARCS) VERTEX Point WITHOUT OVERLAPS > 0.050; + END Parcel; + END Records; +END GeometryArc24. diff --git a/testdata/reference/iox/ili24/arc-above-tolerance.xtf b/testdata/reference/iox/ili24/arc-above-tolerance.xtf new file mode 100644 index 00000000..c55749b6 --- /dev/null +++ b/testdata/reference/iox/ili24/arc-above-tolerance.xtf @@ -0,0 +1,35 @@ + + + + GeometryArc24 + iox-reference + + + + + + + 10.00010.000 + 20.00010.000 + 20.00030.000 + 10.00030.000 + 10.00010.000 + + + + + + + 20.00010.000 + 30.00010.000 + 30.00030.000 + 20.01030.000 + 20.00010.00019.94820.000 + + + + + + diff --git a/testdata/reference/iox/ili24/arc-below-tolerance.xtf b/testdata/reference/iox/ili24/arc-below-tolerance.xtf new file mode 100644 index 00000000..fa0be8fd --- /dev/null +++ b/testdata/reference/iox/ili24/arc-below-tolerance.xtf @@ -0,0 +1,35 @@ + + + + GeometryArc24 + iox-reference + + + + + + + 10.00010.000 + 20.00010.000 + 20.00030.000 + 10.00030.000 + 10.00010.000 + + + + + + + 20.00010.000 + 30.00010.000 + 30.00030.000 + 20.01030.000 + 20.00010.00019.95220.000 + + + + + + diff --git a/testdata/reference/iox/ili24/areareas/GeometryArc24.ili b/testdata/reference/iox/ili24/areareas/GeometryArc24.ili new file mode 100644 index 00000000..6fe19200 --- /dev/null +++ b/testdata/reference/iox/ili24/areareas/GeometryArc24.ili @@ -0,0 +1,11 @@ +INTERLIS 2.4; + +MODEL GeometryArc24 (en) AT "https://example.invalid/iox-reference" VERSION "2026-11-06" = + TOPIC Records = + DOMAIN Point = COORD 0.000 .. 100.000 [INTERLIS.m], 0.000 .. 100.000 [INTERLIS.m]; + CLASS Parcel = + shape : MANDATORY SURFACE WITH (STRAIGHTS, ARCS) VERTEX Point WITHOUT OVERLAPS > 0.050; + SET CONSTRAINT INTERLIS.areAreas(ALL, UNDEFINED, >>shape); + END Parcel; + END Records; +END GeometryArc24. diff --git a/tools/iox-reference-harness/LegacyProbe.java b/tools/iox-reference-harness/LegacyProbe.java index 9607d277..25ea90f2 100644 --- a/tools/iox-reference-harness/LegacyProbe.java +++ b/tools/iox-reference-harness/LegacyProbe.java @@ -30,7 +30,7 @@ final class LegacyProbe { private static final int SAMPLE_INTERVAL_MILLIS = 5; public static void main(String[] args) throws Exception { - measureHeap(Path.of(args[0])); + measureHeap(Path.of(args[0]), args[1].equals("benchmark")); switch (args[1]) { case "read" -> { int exitCode = read(args[2], args[3], Path.of(args[4])); @@ -38,7 +38,7 @@ public static void main(String[] args) throws Exception { System.exit(exitCode); } } - case "validate" -> org.interlis2.validator.Main.main(Arrays.copyOfRange(args, 2, args.length)); + case "validate", "benchmark" -> org.interlis2.validator.Main.main(Arrays.copyOfRange(args, 2, args.length)); default -> throw new IllegalArgumentException("Unknown probe: " + args[1]); } } @@ -129,7 +129,7 @@ private static Map snapshot(IomObject object) { "attributes", attributes); } - private static void measureHeap(Path output) { + private static void measureHeap(Path output, boolean includeGc) { var memory = ManagementFactory.getMemoryMXBean(); var peak = new AtomicLong(); Runnable sample = () -> peak.accumulateAndGet(memory.getHeapMemoryUsage().getUsed(), Math::max); @@ -147,14 +147,34 @@ private static void measureHeap(Path output) { Runtime.getRuntime().addShutdownHook(Thread.ofPlatform().unstarted(() -> { sample.run(); try { - Files.writeString(output, json(fields("heapLimitBytes", memory.getHeapMemoryUsage().getMax(), - "sampledPeakHeapBytes", peak.get(), "heapSampleIntervalMillis", SAMPLE_INTERVAL_MILLIS))); + var measurements = fields("heapLimitBytes", memory.getHeapMemoryUsage().getMax(), + "sampledPeakHeapBytes", peak.get(), "heapSampleIntervalMillis", SAMPLE_INTERVAL_MILLIS); + if (includeGc) { + measurements.put("gcCollectionMillis", gcCollectionMillis()); + } + Files.writeString(output, json(measurements)); } catch (IOException failure) { failure.printStackTrace(System.err); } })); } + private static Long gcCollectionMillis() { + var collectors = ManagementFactory.getGarbageCollectorMXBeans(); + if (collectors.isEmpty()) { + return null; + } + long total = 0; + for (var collector : collectors) { + long millis = collector.getCollectionTime(); + if (millis < 0) { + return null; + } + total += millis; + } + return total; + } + private static Map fields(Object... values) { var map = new LinkedHashMap(); for (int i = 0; i < values.length; i += 2) { diff --git a/tools/iox-reference-harness/README.md b/tools/iox-reference-harness/README.md index f1e2c0b0..5c5c403b 100644 --- a/tools/iox-reference-harness/README.md +++ b/tools/iox-reference-harness/README.md @@ -139,3 +139,37 @@ Byte Heap und bis zu 1'170 Byte temporäre Dateien. Es sind Smoke-Messungen an kleinen Eingaben. RSS, native Allokationen, physisch belegte Blöcke und das Verhalten bei grossen Datenmengen brauchen eigene Messungen. + +## Geometrie und Ressourcen messen + +Der Benchmarkmodus verwendet ein lokales, hashgeprüftes ZIP und explizit +gewählte Fälle. Die [Profile](../../testdata/reference/iox/geometry-cases.json) +erfassen Prüfpunkte, Sollbefunde und Geometriezahlen. Sie enthalten +Gitter mit 100 und 1’000 Polygonen, gemeinsame Ränder, Rundung, +Selbstschnitt sowie Bogen- und `areAreas(ALL, …)`-Proben. + +```sh +python3 -B tools/iox-reference-harness/run.py --benchmark \ + --archive /absolute/ilivalidator-1.15.0.zip --output /tmp/iox-geometry \ + --case area-grid-disjoint-1000 --case area-grid-shared-edges-1000 +``` + +Jeder Aufruf startet die Fälle seriell in frischen JVMs unter `-Xmx2g` +mit beiden Validierungspässen und `--trace`. Standardmässig gelten pro +Aufruf höchstens acht Fälle und pro Fall 30 s sowie 64 MiB abgetasteter +Scratch inklusive Logs. `--max-cases`, `--timeout-seconds` und +`--temp-limit-bytes` setzen die Limits innerhalb fester Obergrenzen. +Der Runner prüft die Eingabegrössen vor dem Start. Bei einem Limitstopp +meldet er Grund, JVM-Exit und Bereinigung. Der Budgettest tastet logische +Dateigrössen ab. Zwischen zwei Stichproben kann die Belegung weiter steigen. +Rohlogs, GC-Log und Bericht bleiben im neuen Ausgabeverzeichnis ausserhalb +des Repositorys. + +Wandzeit, Prozess-CPU und RSS umfassen den JVM-Start. Der abgetastete +Heap und die kumulierte GC-Zeit sind JVM-Messwerte. RSS und CPU stammen, +soweit verfügbar, aus dem OS-Kindprozessstatus. I/O-Mengen und tatsächliche +räumliche Kandidaten bleiben offen. Die erkannten `filesize`-Zeilen im +Trace geben Poolgrössen beim Schliessen an. Sie ergänzen die abgetastete +Temp-Spitze und den Temp-Endbestand. +`PASS` bedeutet weiterhin, dass der Lauf den Sollbefund erfüllt hat, +einschliesslich der erwarteten Zurückweisung eines Negativfalls. diff --git a/tools/iox-reference-harness/geometry.py b/tools/iox-reference-harness/geometry.py new file mode 100644 index 00000000..948e1667 --- /dev/null +++ b/tools/iox-reference-harness/geometry.py @@ -0,0 +1,52 @@ +"""Render the bounded, straight-segment XTF 2.3 geometry profiles.""" + +MAX_OBJECTS = 1024 + + +def grid(count, seed, spacing): + if not 1 <= count <= MAX_OBJECTS or not 1 <= seed <= 5 or spacing not in (2, 3): + raise ValueError("Grid profile is outside the fixed geometry bounds") + for index in range(count): + x = seed + index % 32 * spacing + y = seed + index // 32 * spacing + corners = [(x, y), (x + 2, y), (x + 2, y + 2), (x, y + 2), (x, y)] + yield [(f"{x}.00", f"{y}.00") for x, y in corners] + + +def transfer(case): + profile = case["workload"] + count = profile["objects"] + if not 1 <= count <= MAX_OBJECTS: + raise ValueError("Generated case exceeds the object bound") + spec = case["generate"] + rings = list(grid(count, spec["seed"], spec["spacing"])) if "spacing" in spec else spec["rings"] + if len(rings) != count or profile["polygons"] != count or profile["rings"] != count: + raise ValueError("Geometry profile has inconsistent object, polygon or ring counts") + if (any(len(ring) != 5 or ring[0] != ring[-1] for ring in rings) + or profile["straightSegments"] != 4 * count or profile["arcSegments"] != 0): + raise ValueError("Generated profiles require closed four-segment straight rings") + + model, topic, kind = spec["class"].split(".") + basket = f"{model}.{topic}" + tag = f"{basket}.{kind}" + lines = [ + '', + '', + ' ', + " ", + f' ', + " ", + " ", + " ", + f' <{basket} BID="parcel-basket">', + ] + for index, ring in enumerate(rings, start=1): + lines.extend([ + f' <{tag} TID="parcel-{index}">', + " ", + *(f" {x}{y}" for x, y in ring), + " ", + f" ", + ]) + lines.extend([f" ", " ", "", ""]) + return "\n".join(lines).encode("utf-8") diff --git a/tools/iox-reference-harness/run.py b/tools/iox-reference-harness/run.py index 1dbd7a47..70d47056 100644 --- a/tools/iox-reference-harness/run.py +++ b/tools/iox-reference-harness/run.py @@ -2,13 +2,16 @@ """Run the pinned IOX and ilivalidator reference cases in isolated JVMs.""" import argparse +from contextlib import suppress import hashlib import json +import math import os from pathlib import Path import platform import re import shutil +import signal import subprocess import sys import tempfile @@ -20,15 +23,26 @@ JAVA_SOURCE = HARNESS / "LegacyProbe.java" LOCK_FILE = HARNESS / "legacy-lock.json" CASES_FILE = CORPUS / "cases.json" +GEOMETRY_CASES_FILE = CORPUS / "geometry-cases.json" +GEOMETRY_SOURCE = HARNESS / "geometry.py" TEMP_SAMPLE_MS = 5 CASE_TIMEOUT_SECONDS = 30 HEAP_LIMIT_BYTES = 2 * 1024**3 +BENCHMARK_INPUT_BYTES = 16 * 1024**2 +STOP_GRACE_SECONDS = 0.25 +POOL_CLOSE = re.compile( + r"Info: close\(\): ([^:]+): (?:size \d+, )?filesize (\d+)" + r"(?:, updates \d+,)? <.+> \((?:BTree|BTreeImpl|ObjPoolImpl2)\.java:\d+\)" +) +FATAL_LOG = re.compile(r"OutOfMemoryError|No space left on device|A fatal error has been detected", + re.IGNORECASE) JVM_OPTIONS = [ "-Xmx2g", "-Duser.language=en", "-Duser.country=US", "-Dfile.encoding=UTF-8", "-Dstdout.encoding=UTF-8", "-Dstderr.encoding=UTF-8", "-Djavax.xml.accessExternalDTD=", "-Djavax.xml.accessExternalSchema=", ] +GC_LOG_OPTION = "-Xlog:gc,gc+heap=debug,gc+init,gc+heap+exit:file=../gc.log:uptimemillis,level,tags:filecount=0" ENV = {key: value for key, value in os.environ.items() if key not in { "CLASSPATH", "JAVA_TOOL_OPTIONS", "JDK_JAVA_OPTIONS", "JDK_JAVAC_OPTIONS", "_JAVA_OPTIONS", }} @@ -98,32 +112,105 @@ def temp_bytes(directory): return total -def run_probe(argv, work, temp): +def run_probe(argv, work, temp, limits=None): env = ENV | {key: str(temp) for key in ("TMPDIR", "TMP", "TEMP")} + cwd = work + timeout = CASE_TIMEOUT_SECONDS + resource_platform = platform.system() if limits else None + use_wait4 = resource_platform in ("Darwin", "Linux") and hasattr(os, "wait4") + if limits: + env.pop("ILI_LEGACYAREAREAS", None) + cwd = temp + timeout = limits["timeoutSeconds"] started = time.monotonic() peak_temp = 0 - timed_out = False + peak_scratch = 0 + stop_reason = None + stop_started = None + monitor_error = None + signals = [] + usage = None with (work / "stdout.log").open("wb") as stdout, (work / "stderr.log").open("wb") as stderr: - with subprocess.Popen(argv, cwd=work, env=env, stdout=stdout, stderr=stderr) as child: + with subprocess.Popen(argv, cwd=cwd, env=env, stdout=stdout, stderr=stderr) as child: try: - while child.poll() is None: - peak_temp = max(peak_temp, temp_bytes(temp)) - if time.monotonic() - started >= CASE_TIMEOUT_SECONDS: - timed_out = True - child.kill() + while True: + if use_wait4: + pid, status, child_usage = os.wait4(child.pid, os.WNOHANG) + if pid: + child.returncode = os.waitstatus_to_exitcode(status) + usage = child_usage + else: + child.poll() + if not limits and child.returncode is not None: break + current_temp = temp_bytes(temp) + peak_temp = max(peak_temp, current_temp) + if limits: + log_bytes = os.fstat(stdout.fileno()).st_size + os.fstat(stderr.fileno()).st_size + for name in ("gc.log", "heap.json"): + with suppress(FileNotFoundError): + log_bytes += (work / name).stat().st_size + peak_scratch = max(peak_scratch, current_temp + log_bytes) + if peak_scratch > limits["tempLimitBytes"]: + stop_reason = stop_reason or "temp_limit" + now = time.monotonic() + if now - started >= timeout and (limits or child.returncode is None): + stop_reason = stop_reason or "timeout" + if child.returncode is not None: + break + if stop_reason: + if not limits: + child.kill() + break + # Popen.send_signal can reap the child before wait4 collects its usage. + if stop_started is None: + stop_started = now + with suppress(ProcessLookupError): + if use_wait4: + os.kill(child.pid, signal.SIGTERM) + else: + child.terminate() + signals.append("terminate") + elif signals == ["terminate"] and now - stop_started >= STOP_GRACE_SECONDS: + with suppress(ProcessLookupError): + if use_wait4: + os.kill(child.pid, signal.SIGKILL) + else: + child.kill() + signals.append("kill") time.sleep(TEMP_SAMPLE_MS / 1000) + except KeyboardInterrupt: + if not limits: + raise + stop_reason = "interrupted" + except OSError as failure: + if not limits: + raise + stop_reason = stop_reason or "monitor_error" + monitor_error = str(failure) finally: - if child.poll() is None: + if child.returncode is None: child.kill() + if limits: + signals.append("kill") exit_code = child.wait() elapsed = time.monotonic() - started temp_at_exit = temp_bytes(temp) - return { - "exitCode": exit_code, "timedOut": timed_out, "wallMillis": round(elapsed * 1000, 2), + result = { + "exitCode": exit_code, "timedOut": stop_reason == "timeout", "wallMillis": round(elapsed * 1000, 2), "sampledPeakTempBytes": max(peak_temp, temp_at_exit), "tempAtExitBytes": temp_at_exit, "tempSampleIntervalMillis": TEMP_SAMPLE_MS, } + if limits: + result.update(stopReason=stop_reason, stopSignals=signals, monitorError=monitor_error, + sampledPeakScratchBytes=peak_scratch, + processCpuMillis=None, peakRssBytes=None, processUsageSource=None, + processReadBytes=None, processWrittenBytes=None) + if usage: + result.update(processCpuMillis=round((usage.ru_utime + usage.ru_stime) * 1000, 2), + peakRssBytes=usage.ru_maxrss * (1024 if resource_platform == "Linux" else 1) or None, + processUsageSource=f"{resource_platform} wait4") + return result def reader_check(case, stdout, stderr, result): @@ -168,12 +255,15 @@ def validator_check(case, stdout, stderr, versions, result): result["mismatches"].append("Unexpected validator output outside Info, Error, and Warning lines") if result["missingInfo"]: result["mismatches"].append("Missing validator phase, coverage, or version messages") + forbidden = [marker for marker in case.get("forbiddenInfo", []) if marker in info] + if forbidden: + result["mismatches"].append(f"Unexpected validation coverage: {forbidden}") -def run_case(case, output, classpath, versions): +def run_case(case, output, classpath, versions, limits=None, java="java", generated=None): work = output / case["id"] temp = work / "temp" - home = work / "home" + home = (temp if limits else work) / "home" temp.mkdir(parents=True) home.mkdir() inputs = work / "inputs" @@ -181,13 +271,21 @@ def run_case(case, output, classpath, versions): model = inputs / Path(case["model"]).name xtf = inputs / Path(case["xtf"]).name shutil.copyfile(CORPUS / case["model"], model) - shutil.copyfile(CORPUS / case["xtf"], xtf) + if generated is None: + shutil.copyfile(CORPUS / case["xtf"], xtf) + else: + xtf.write_bytes(generated) heap_log = work / "heap.json" - argv = ["java", *JVM_OPTIONS, f"-Duser.home={home}", f"-Djava.io.tmpdir={temp}", - "-cp", classpath, "LegacyProbe", str(heap_log), case["mode"]] + options = [*JVM_OPTIONS, f"-Duser.home={home}", f"-Djava.io.tmpdir={temp}"] + if limits: + options.append(GC_LOG_OPTION) + mode = "benchmark" if limits else case["mode"] + argv = [java, *options, "-cp", classpath, "LegacyProbe", str(heap_log), mode] if case["mode"] == "read": argv += [case["version"], str(model), str(xtf)] elif case["mode"] == "validate": + if limits: + argv.append("--trace") argv += ["--plugins", str(work / "plugins"), "--modeldir", str(model.parent), "--models", model.stem, str(xtf)] (work / "plugins").mkdir() @@ -200,19 +298,46 @@ def run_case(case, output, classpath, versions): result["inputSha256"] = {case["model"]: sha256(model), case["xtf"]: sha256(xtf)} if "events" in case: result["inputSha256"][case["events"]] = sha256(CORPUS / case["events"]) - result.update(run_probe(argv, work, temp)) + try: + result.update(run_probe(argv, work, temp, limits)) + finally: + if limits: + try: + shutil.rmtree(temp) + except OSError as failure: + result["mismatches"].append(f"Cannot clean benchmark scratch: {failure}") + result["scratchRemoved"] = not temp.exists() if heap_log.exists(): - heap = read_json(heap_log) + try: + heap = read_json(heap_log) + except (OSError, ValueError): + if not limits: + raise + heap = None result["heap"] = heap - if heap.get("heapLimitBytes") != HEAP_LIMIT_BYTES: + if not isinstance(heap, dict) or heap.get("heapLimitBytes") != HEAP_LIMIT_BYTES: result["mismatches"].append(f"Expected JVM heap limit of {HEAP_LIMIT_BYTES} bytes (-Xmx2g)") + elif limits and "gcCollectionMillis" not in heap: + result["mismatches"].append("Missing GC collection measurement status") else: + if limits: + result["heap"] = None result["mismatches"].append("Missing JVM heap sample") - if result["timedOut"]: + if limits and result["stopReason"]: + result["mismatches"].append(f"Benchmark stopped: {result['stopReason']}") + elif result["timedOut"]: result["mismatches"].append(f"Exceeded {CASE_TIMEOUT_SECONDS} s process timeout") - stdout = (work / "stdout.log").read_text(encoding="utf-8") - stderr = (work / "stderr.log").read_text(encoding="utf-8") - result["logSha256"] = {name: sha256(work / name) for name in ("stdout.log", "stderr.log")} + errors = "replace" if limits else "strict" + stdout = (work / "stdout.log").read_text(encoding="utf-8", errors=errors) + stderr = (work / "stderr.log").read_text(encoding="utf-8", errors=errors) + logs = ["stdout.log", "stderr.log"] + if limits: + if (work / "gc.log").is_file(): + logs.append("gc.log") + elif not result["stopReason"]: + result["mismatches"].append("Missing JVM GC log") + benchmark_result(case, model, xtf, stdout, stderr, result) + result["logSha256"] = {name: sha256(work / name) for name in logs} if case["mode"] == "read": reader_check(case, stdout, stderr, result) else: @@ -226,20 +351,108 @@ def run_case(case, output, classpath, versions): return result +def benchmark_result(case, model, xtf, stdout, stderr, result): + lines = stdout.splitlines() + stderr.splitlines() + pools = [match.groups() for line in lines if (match := POOL_CLOSE.fullmatch(line))] + result["poolCloseSizes"] = [{"pool": pool, "logicalBytes": int(size)} for pool, size in pools] or None + missing_pools = set(case.get("requiredPools", [])) - {pool for pool, _ in pools} + if missing_pools: + result["mismatches"].append(f"Missing required trace pools: {sorted(missing_pools)}") + result["inputBytes"] = {case["model"]: model.stat().st_size, case["xtf"]: xtf.stat().st_size} + result["workload"] = case.get("workload", {"profile": "reference-smoke-control"}) + result["verifiedInfo"] = [marker for marker in + ["first validation pass...", "second validation pass...", *case["requiredInfo"]] + if f"Info: {marker}" in lines] + fatal = FATAL_LOG.search(stdout + "\n" + stderr) + if fatal: + result["mismatches"].append(f"Fatal JVM or storage failure: {fatal.group()}") + + +def benchmark_limits(parser, args, selected): + import geometry + + if not args.archive or not args.archive.is_file(): + parser.error("BLOCKED_INPUT: --benchmark requires a local release ZIP via --archive") + output = args.output or Path(tempfile.gettempdir()) + if output.resolve().is_relative_to(HARNESS.parents[1]): + parser.error("benchmark output must be outside the repository") + if not args.case or any(case["mode"] != "validate" for case in selected): + parser.error("--benchmark requires explicit validator --case selections") + if not math.isfinite(args.timeout_seconds) or not 0 < args.timeout_seconds <= 300: + parser.error("--timeout-seconds must be greater than 0 and at most 300") + if not 0 < args.temp_limit_bytes <= 1024**3: + parser.error("--temp-limit-bytes must be between 1 and 1073741824") + if not len(selected) <= args.max_cases <= 32: + parser.error("--max-cases must cover the selected cases and be at most 32") + generated = {} + input_bytes = 0 + for case in selected: + input_bytes += (CORPUS / case["model"]).stat().st_size + if "generate" in case: + transfer = geometry.transfer(case) + generated[case["id"]] = transfer + input_bytes += len(transfer) + else: + input_bytes += (CORPUS / case["xtf"]).stat().st_size + if input_bytes > BENCHMARK_INPUT_BYTES: + parser.error("benchmark inputs exceed the 16 MiB preflight bound") + if args.archive.stat().st_size > BENCHMARK_INPUT_BYTES: + parser.error("benchmark release ZIP exceeds the 16 MiB preflight bound") + limits = {"timeoutSeconds": args.timeout_seconds, "tempLimitBytes": args.temp_limit_bytes, + "maxCases": args.max_cases, "maxInputBytes": BENCHMARK_INPUT_BYTES} + return limits, generated + + +def benchmark_jdk(output, report, env): + settings = setup_process(["java", "-XshowSettings:properties", "-version"], output, report, env) + match = re.search(r"^\s*java\.home = (.+)$", settings, re.MULTILINE) + if not match: + raise ValueError("Cannot identify benchmark Java installation") + jdk = Path(match[1]).resolve() + java = "java.exe" if os.name == "nt" else "java" + javac = "javac.exe" if os.name == "nt" else "javac" + vm = "bin/server/jvm.dll" if os.name == "nt" else ( + "lib/server/libjvm.dylib" if platform.system() == "Darwin" else "lib/server/libjvm.so") + report["jdkSha256"] = {name: sha256(jdk / name) for name in + ("release", f"bin/{java}", f"bin/{javac}", "lib/modules", vm)} + return str(jdk / "bin" / java), str(jdk / "bin" / javac) + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--archive", type=Path, help="verify and use this local release ZIP without downloading") parser.add_argument("--output", type=Path, help="new directory for reports (default: a fresh temp directory)") parser.add_argument("--case", action="append", help="run only this case ID (repeatable)") + parser.add_argument("--benchmark", action="store_true", help="measure explicitly selected validator cases") + parser.add_argument("--timeout-seconds", type=float, help="benchmark JVM timeout (default: 30)") + parser.add_argument("--temp-limit-bytes", type=int, help="benchmark writable scratch limit (default: 64 MiB)") + parser.add_argument("--max-cases", type=int, help="maximum cases per benchmark invocation (default: 8)") args = parser.parse_args() + if not args.benchmark and any(value is not None for value in + (args.timeout_seconds, args.temp_limit_bytes, args.max_cases)): + parser.error("benchmark budget options require --benchmark") cases = read_json(CASES_FILE) + source_files = [Path(__file__), JAVA_SOURCE, LOCK_FILE, CASES_FILE] + if args.benchmark: + cases += read_json(GEOMETRY_CASES_FILE) + source_files += [GEOMETRY_CASES_FILE, GEOMETRY_SOURCE] ids = {case["id"] for case in cases} if not ids or len(ids) != len(cases): parser.error("case inventory must be nonempty with unique IDs") if args.case and set(args.case) - ids: parser.error(f"unknown case IDs: {', '.join(sorted(set(args.case) - ids))}") selected = [case for case in cases if not args.case or case["id"] in args.case] + limits = None + generated = {} + if args.benchmark: + if args.timeout_seconds is None: + args.timeout_seconds = 30 + if args.temp_limit_bytes is None: + args.temp_limit_bytes = 64 * 1024**2 + if args.max_cases is None: + args.max_cases = 8 + limits, generated = benchmark_limits(parser, args, selected) archive_arg = str(args.archive.resolve()) if args.archive else None try: if args.output: @@ -256,28 +469,41 @@ def main(): report = { "schema": 1, "python": sys.version.splitlines()[0], "platform": platform.platform(), "invocation": [sys.executable, str(Path(__file__).resolve()), *sys.argv[1:]], - "sourceSha256": {path.name: sha256(path) for path in (Path(__file__), JAVA_SOURCE, LOCK_FILE, CASES_FILE)}, + "sourceSha256": {path.name: sha256(path) for path in source_files}, "selectedCases": [case["id"] for case in selected], "setup": [], "cases": [], "versions": lock["versions"], "logSha256Scope": "raw-output-of-this-run", "status": "FAIL", } + if limits: + report.update(mode="benchmark", budgets=limits, architecture=platform.machine(), + jvmOptions=[*JVM_OPTIONS, GC_LOG_OPTION], measurementScope={ + "wallMillis": "JVM launch through exit observation, including model compilation and shutdown", + "processCpuMillis and peakRssBytes": "whole JVM process, including startup and shutdown", + "heap": "JVM used heap sampled from probe startup through shutdown; GC time is a JVM estimate", + "sampledPeakTempBytes": "named logical files in isolated JVM cwd, home and tmpdir", + "sampledPeakScratchBytes": "temp bytes plus output logs and heap.json; sampled, not a filesystem quota", + "poolCloseSizes": "logical file sizes from recognized CLI trace records at pool close", + }) try: setup_temp = output / "setup-temp" setup_temp.mkdir() setup_env = ENV | {key: str(setup_temp) for key in ("TMPDIR", "TMP", "TEMP")} - java_version = setup_process(["java", "-version"], output, report, setup_env) - javac_version = setup_process(["javac", "-version"], output, report, setup_env) + java, javac = benchmark_jdk(output, report, setup_env) if limits else ("java", "javac") + java_version = setup_process([java, "-version"], output, report, setup_env) + javac_version = setup_process([javac, "-version"], output, report, setup_env) if not re.search(r'version "25(?=[."+-])', java_version) or not re.match(r'javac 25(?:[.+-]|$)', javac_version): raise ValueError("Java 25 and javac 25 are required on PATH") entrypoint = legacy_jars(lock, archive_arg, output, report, setup_env) classes = output / "classes" classes.mkdir() - setup_process(["javac", "--release", "25", "-encoding", "UTF-8", "-cp", str(entrypoint), + setup_process([javac, "--release", "25", "-encoding", "UTF-8", "-cp", str(entrypoint), "-d", str(classes), str(JAVA_SOURCE)], output, report, setup_env) classpath = os.pathsep.join(map(str, (classes, entrypoint))) for case in selected: - result = run_case(case, output, classpath, lock["versions"]) + result = run_case(case, output, classpath, lock["versions"], limits, java, generated.get(case["id"])) report["cases"].append(result) print(f"{result['status']:5} {result['id']} (exit {result['exitCode']})") + if result.get("stopReason") == "interrupted": + break if all(result["status"] != "FAIL" for result in report["cases"]): report["status"] = "PASS" except (OSError, ValueError, subprocess.TimeoutExpired) as failure: diff --git a/tools/iox-reference-harness/test_run.py b/tools/iox-reference-harness/test_run.py index 98770b12..782a233a 100644 --- a/tools/iox-reference-harness/test_run.py +++ b/tools/iox-reference-harness/test_run.py @@ -1,5 +1,10 @@ """Small regression checks for the validator output oracle.""" +import os +from pathlib import Path +import signal +import sys +from tempfile import TemporaryDirectory import unittest import run @@ -30,5 +35,49 @@ def test_unexpected_stderr_fails_even_when_exit_and_diagnostics_match(self): self.assertTrue(invalid["mismatches"]) +class BenchmarkLimitsTest(unittest.TestCase): + def test_sampled_scratch_limit_terminates_a_running_child(self): + with TemporaryDirectory() as directory: + work = Path(directory) + scratch = work / "temp" + scratch.mkdir() + limits = {"timeoutSeconds": 5, "tempLimitBytes": 16} + script = "from pathlib import Path; import time; Path('spill').write_bytes(b'x' * 128); time.sleep(10)" + result = run.run_probe([sys.executable, "-c", script], work, scratch, limits) + self.assertEqual(result["stopReason"], "temp_limit") + self.assertIn("terminate", result["stopSignals"]) + self.assertNotEqual(result["exitCode"], 0) + self.assertEqual(result["sampledPeakTempBytes"], 128) + self.assertEqual(result["tempAtExitBytes"], 128) + + @unittest.skipUnless(os.name == "posix", "SIGTERM and SIGKILL escalation requires POSIX") + def test_temp_limit_kills_a_child_that_ignores_termination(self): + script = """\ +from pathlib import Path +import signal +import time + +signal.signal(signal.SIGTERM, signal.SIG_IGN) +Path("spill").write_bytes(b"x" * 128) +time.sleep(10) +""" + with TemporaryDirectory() as directory: + work = Path(directory) + scratch = work / "temp" + scratch.mkdir() + limits = {"timeoutSeconds": 5, "tempLimitBytes": 16} + result = run.run_probe([sys.executable, "-c", script], work, scratch, limits) + self.assertEqual(result["stopReason"], "temp_limit") + self.assertEqual(result["stopSignals"], ["terminate", "kill"]) + self.assertEqual(result["exitCode"], -signal.SIGKILL) + + def test_pool_size_uses_recognized_close_file_size(self): + prefix = "Info: close(): LinkPool KEY: " + suffix = " (BTree.java:396)" + match = run.POOL_CLOSE.fullmatch(prefix + "size 3, filesize 4096" + suffix) + self.assertEqual(match.groups(), ("LinkPool KEY", "4096")) + self.assertIsNone(run.POOL_CLOSE.fullmatch(prefix + "size 3" + suffix)) + + if __name__ == "__main__": unittest.main()