diff --git a/build/interlis/JvmBuildInputs.java b/build/interlis/JvmBuildInputs.java index 8ea27d08..49d1097d 100644 --- a/build/interlis/JvmBuildInputs.java +++ b/build/interlis/JvmBuildInputs.java @@ -12,7 +12,7 @@ public final class JvmBuildInputs { public static final List MODULES = List.of("interlis-model", "interlis-compiler", "interlis-workspace", "interlis-repository", "interlis-projection", "interlis-documentation", - "interlis-transfer", "interlis-wasm-bridge", "interlis-compiler-cli", "interlis-lsp", + "interlis-transfer", "interlis-xtf23", "interlis-wasm-bridge", "interlis-compiler-cli", "interlis-lsp", "interlis-compiler-gui", "interlis-verification"); public static void requireProject(Path root) throws Exception { diff --git a/build/interlis/ModulePathContract.java b/build/interlis/ModulePathContract.java index 93228025..9f2c1728 100644 --- a/build/interlis/ModulePathContract.java +++ b/build/interlis/ModulePathContract.java @@ -15,6 +15,13 @@ public static String name(String project) { public static void requireOwnDescriptor(String project, ModuleDescriptor descriptor) { if(descriptor.isAutomatic()||descriptor.isOpen()||!descriptor.name().equals(name(project))) throw new IllegalStateException("Own runtime module must be explicit and encapsulated: "+project); + if(project.equals("interlis-xtf23")) { + var required=descriptor.requires().stream().map(ModuleDescriptor.Requires::name).collect(java.util.stream.Collectors.toSet()); + if(!required.equals(Set.of("java.base","java.xml","guru.interlis.model","guru.interlis.transfer")) + ||descriptor.exports().size()!=1||descriptor.exports().stream().anyMatch(export-> + export.isQualified()||!export.source().equals("guru.interlis.xtf23"))||!descriptor.opens().isEmpty()) + throw new IllegalStateException("Unexpected XTF 2.3 module boundary: "+descriptor); + } } public static void run(Path root)throws Exception { for(String project:JvmBuildInputs.MODULES)if(!Files.isRegularFile(root.resolve(project+"/src/main/java/module-info.java")))throw new IllegalStateException("Missing own module descriptor: "+project); @@ -26,11 +33,12 @@ public static void run(Path root)throws Exception { if(!result.explicitDescriptor()||!result.moduleName().orElseThrow().equals(name(project)))throw new IllegalStateException("Wrong own module: "+project); own.add(name(project)); var runtime=ModuleArtifacts.runtime(root,project,outcome); - if(project.equals("interlis-transfer")) { + if(project.equals("interlis-transfer")||project.equals("interlis-xtf23")) { String prefix="maven/guru.interlis/", version="/"+JvmBuildInputs.projectVersion(root); - var expected=Set.of(prefix+"interlis-transfer"+version,prefix+"interlis-model"+version); + var expected=new HashSet<>(Set.of(prefix+"interlis-transfer"+version,prefix+"interlis-model"+version)); + if(project.equals("interlis-xtf23"))expected.add(prefix+project+version); var actual=runtime.stream().map(ModuleArtifacts.Artifact::coordinates).collect(java.util.stream.Collectors.toSet()); - if(!actual.equals(expected))throw new IllegalStateException("Unexpected transfer runtime closure: "+actual); + if(!actual.equals(expected))throw new IllegalStateException("Unexpected runtime closure for "+project+": "+actual); } for(var artifact:runtime) { Path prior=files.putIfAbsent(artifact.fileName(),artifact.file()); @@ -71,6 +79,7 @@ public static void run(Path root)throws Exception { "guru.interlis.projection","guru.interlis.projection.DiagramProjector", "guru.interlis.documentation","guru.interlis.documentation.DocumentationProjector", "guru.interlis.transfer","guru.interlis.transfer.TransferPlanner", + "guru.interlis.xtf23","guru.interlis.xtf23.Xtf23Reader", "guru.interlis.wasm.bridge","guru.interlis.bridge.BridgeDispatcher"); String extra=String.join("\n",own.stream().filter(entrypoints::containsKey).map(name-> "if (named && !"+entrypoints.get(name)+".class.getModule().getName().equals(\""+name+"\")) throw new AssertionError(\"Wrong public entrypoint module\");").toList()); diff --git a/docs/architecture.md b/docs/architecture.md index 7eef1cfd..94e654c1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -21,13 +21,14 @@ Die zugehörigen POMs beschreiben die konkreten Abhängigkeiten und Scopes. | `interlis-projection` | compiler, model | Diagramm-DTOs und reine Projektion | | `interlis-documentation` | model, projection | Lesende Dokumentationsprojektion und Rendering | | `interlis-transfer` | model | Formatunabhängige Transferpläne, Objektwerte und Streaming-Protokoll | +| `interlis-xtf23` | transfer, model | XML-Adapter für INTERLIS 2.3 | | `interlis-compiler-cli` | compiler, model, repository, workspace, projection | CLI, Prozessgrenzen, Präsentation und Ausgabe | | `interlis-compiler-gui` | model, compiler, workspace, repository | Desktop-Modellprüfung, Diagnosen, Abhängigkeiten und Repositoryexplorer | | `interlis-lsp` | workspace, compiler, projection, model, repository, documentation | LSP-/Editorhost, Scheduling und Ausgaben | | `interlis-wasm-bridge` | workspace, compiler, projection, model | Gemeinsames JSON-Protokoll und Sitzungen | | `interlis-verification` | compiler, model, workspace, repository, wasm-bridge | Prüfwerkzeuge ausserhalb der SDK-Auslieferung | -Alle zwölf JVM-Module werden über denselben Builder erzeugt. Die +Alle dreizehn JVM-Module werden über denselben Builder erzeugt. Die jeweils ausgewählten JPMS-Closures stehen im [Buildvertrag](development/build.md). TeaVM und Web Image konsumieren denselben expliziten JVM-Snapshot. Browser und Editor teilen den neutralen diff --git a/docs/design/transfer-validation-database.md b/docs/design/transfer-validation-database.md index 18bcf54d..32c43b8b 100644 --- a/docs/design/transfer-validation-database.md +++ b/docs/design/transfer-validation-database.md @@ -1,6 +1,6 @@ # Entwurf für Transfer, Validierung und Datenbanken -Der formatunabhängige Transferkern steht in `interlis-transfer` für INTERLIS 2.3 und 2.4 zur Verfügung. Formatadapter, Datenvalidierung und Datenbankabbildung sind geplant. Die API-Skizzen beschreiben den vorgesehenen Ausbau. +Der formatunabhängige Transferkern steht in `interlis-transfer` für INTERLIS 2.3 und 2.4 zur Verfügung. `interlis-xtf23` liest und schreibt XTF-2.3-FULL-Transfers objektweise. Weitere Formatadapter, Datenvalidierung und Datenbankabbildung sind geplant. Die API-Skizzen beschreiben den vorgesehenen Ausbau. Die Verantwortung wird entlang fachlicher Produkte geschnitten: @@ -103,6 +103,12 @@ Ein `TransferObject` hält Identität, Typbezug und Attribut-/Rollenwerte eines Der Kern vergleicht die Zeichenwerte innerhalb dieser Namensräume unabhängig von der Herkunft `SOURCE` oder `SYNTHETIC`. Die OID-Regeln für INITIAL und UPDATE gehören zum jeweiligen Transferprofil. Der XTF-2.3-Adapter muss für FULL zusätzlich BID und TID gemeinsam prüfen, auch wenn die gleichlautende Kennung zuerst als TID erscheint. In XTF 2.4 gilt die transferweite Eindeutigkeit der FULL-TIDs und die allgemeine Objektstromregel für Behälter- und Objektidentifikationen. Die Akzeptanz gleicher Zeichenwerte in den getrennten Namensräumen des Kerns trifft keine Aussage über die Gültigkeit einer konkreten XTF-Codierung. [Gepinnte Normbelege und P4.0-Befunde](../reference/iox-legacy.md#normbelege) +`Xtf23Reader` und `Xtf23Writer` verwenden vom Aufrufer verwaltete Streams. OIDSPACE-Deklarationen bleiben in Quellreihenfolge am Adapter verfügbar. Der FULL-Adapter liest und schreibt BLACKBOX XML und BINARY, OID-Attributwerte sowie CLASS-, STRUCTURE- und ATTRIBUTE-Pfadwerte. Modellgebundene Linienattribute bleiben an jeder einzelnen Randlinie von SURFACE und AREA erhalten. Nichtleere ALIAS-Tabellen, inkrementelle Transfers, CLIPPED-Geometrien und benutzerdefinierte Linienformen meldet der Adapter als `UNSUPPORTED`. Er prüft die XML- und Wire-Struktur. Datenconstraints und Geometrietopologie gehören zur späteren Validierung. + +Die LINEATTR-Abnahme prüft den Reader-/Writer-Roundtrip gemäss INTERLIS 2.3 §§3.3.11.12–13. In ilivalidator 1.15.0 meldet der Standardreader von IOX 1.24.4 bei dieser Kodierung `expected segment type`. Mit `ILI_XTF23_READER=ch.interlis.iom_j.xtf.Xtf23Reader0` gelangt die Datei bis zur Polygonprüfung, die `Lineattributes not supported` meldet. Beide Validatorläufe liefern Exit 1. + +Der XTF-2.3-Header führt mindestens ein Modell. Ein leerer Datenbereich ist zulässig. Ein vorhandener BAG- oder LIST-Wrapper enthält mindestens eine Struktur. XTF 2.3 lässt undefinierte Attribute aus und erhält leeren TEXT als eigenen Wert. + Das Datenmodell unterscheidet mindestens: - nicht vorhandenen Slot beziehungsweise Wert; diff --git a/docs/development/build.md b/docs/development/build.md index 0e29fd6b..d2cd5ee3 100644 --- a/docs/development/build.md +++ b/docs/development/build.md @@ -1,6 +1,6 @@ # Build und Toolchains -Jenesis 0.18.1 erzeugt die zwölf JVM-Module mit Java 25 ohne Preview. Die POMs +Jenesis 0.18.1 erzeugt die dreizehn JVM-Module mit Java 25 ohne Preview. Die POMs definieren Group, Artifact, Version, Scopes und Pins. Der Upstreampin gehört zu den [Jenesis-Quellen](../../build/JENESIS-UPSTREAM.json). Der gepinnte Gradle-Wrapper baut ausschliesslich TeaVM und GraalVM Web Image. diff --git a/docs/reference/iox-legacy.md b/docs/reference/iox-legacy.md index 23820bb6..bb71849e 100644 --- a/docs/reference/iox-legacy.md +++ b/docs/reference/iox-legacy.md @@ -148,6 +148,21 @@ abgeschalteten Regeln. [Shapefile #45][wkf-45], [Shapefile #46][wkf-46], [GeoPackage-Tests][wkf-gpkg], [Db2Ili #27][wkf-27]. +### Linienattribute in XTF 2.3 + +Das Referenzhandbuch 2.3 beschreibt `LINEATTR` vor dem Startsegment +(§ 3.3.11.12) und verlangt das Strukturelement in jeder Randlinie einer +entsprechend modellierten SURFACE oder AREA (§ 3.3.11.13). Mit ilivalidator +1.15.0 und iox-ili 1.24.4 endet ein so kodierter Transfer im Standardreader +mit Exit 1 und `expected segment type`. Mit +`ILI_XTF23_READER=ch.interlis.iom_j.xtf.Xtf23Reader0` erreicht derselbe +Transfer die Geometriekonvertierung und endet mit Exit 1 und +`Lineattributes not supported`. Die Diagnosen belegen Grenzen der +Referenzwerkzeuge. Der Adapter prüft die Kodierung und die Werterhaltung +durch Roundtrips. +[Standardreader](https://github.com/claeis/iox-ili/blob/8e966a086a72e7e2990a312b1229ee5878fd5994/src/main/java/ch/interlis/iom_j/xtf/Xtf23Reader.java#L1200), +[Geometriekonvertierung](https://github.com/claeis/iox-ili/blob/8e966a086a72e7e2990a312b1229ee5878fd5994/src/main/java/ch/interlis/iox_j/jts/Iox2jtsext.java#L192). + ## Validierungsphasen und Rollen ilivalidator liest die Eingaben und gegebenenfalls Referenzdaten, steuert diff --git a/interlis-transfer/src/main/java/guru/interlis/transfer/TransferGeometry.java b/interlis-transfer/src/main/java/guru/interlis/transfer/TransferGeometry.java index 217e22a8..8e20b51e 100644 --- a/interlis-transfer/src/main/java/guru/interlis/transfer/TransferGeometry.java +++ b/interlis-transfer/src/main/java/guru/interlis/transfer/TransferGeometry.java @@ -43,13 +43,22 @@ record MultiCoordinate(List coordinates) implements TransferGeometry /** * A polyline with its parts in source order. All parts have the same dimension. * An empty list represents a supplied line with no parts, as in an empty extract. + * + * @param parts nonnull immutable sequence of curve parts + * @param lineAttributes nonnull optional model-bound structure for this line */ - record Polyline(List parts) implements TransferGeometry { + record Polyline(List parts, Optional lineAttributes) implements TransferGeometry { public Polyline { parts = List.copyOf(parts); + Objects.requireNonNull(lineAttributes, "lineAttributes"); requireHomogeneousDimensions(parts, CurvePart::dimensions); } + /** Creates a line without a line-attribute structure. */ + public Polyline(List parts) { + this(parts, Optional.empty()); + } + private int dimensions() { return parts.isEmpty() ? 0 : parts.getFirst().dimensions(); } diff --git a/interlis-transfer/src/main/java/guru/interlis/transfer/TransferObject.java b/interlis-transfer/src/main/java/guru/interlis/transfer/TransferObject.java index 7761fdb0..672cbb21 100644 --- a/interlis-transfer/src/main/java/guru/interlis/transfer/TransferObject.java +++ b/interlis-transfer/src/main/java/guru/interlis/transfer/TransferObject.java @@ -39,7 +39,7 @@ public Optional value(SymbolRef slot) { private static void requireSnapshot(TransferValue.Structured data) { var snapshot = data.type().snapshot(); var seen = Collections.newSetFromMap(new IdentityHashMap()); - var pending = new ArrayDeque>(); + var pending = new ArrayDeque>(); pending.push(data.values().values().iterator()); while (!pending.isEmpty()) { var fields = pending.peek(); @@ -61,6 +61,27 @@ private static void requireSnapshot(TransferValue.Structured data) { pending.push(repeated.values().iterator()); } } + case TransferGeometry.Polyline line -> { + if (seen.add(line)) { + pending.push(line.lineAttributes().stream().iterator()); + } + } + case TransferGeometry.MultiPolyline multi -> { + if (seen.add(multi)) { + pending.push(multi.lines().iterator()); + } + } + case TransferGeometry.Surface surface -> { + if (seen.add(surface)) { + pending.push(surface.shell().lines().iterator()); + pending.push(surface.holes().stream().flatMap(ring -> ring.lines().stream()).iterator()); + } + } + case TransferGeometry.MultiSurface multi -> { + if (seen.add(multi)) { + pending.push(multi.surfaces().iterator()); + } + } default -> { } } } diff --git a/interlis-transfer/src/main/java/guru/interlis/transfer/TransferPlanner.java b/interlis-transfer/src/main/java/guru/interlis/transfer/TransferPlanner.java index 8104687b..62624721 100644 --- a/interlis-transfer/src/main/java/guru/interlis/transfer/TransferPlanner.java +++ b/interlis-transfer/src/main/java/guru/interlis/transfer/TransferPlanner.java @@ -225,7 +225,7 @@ private void projectObject(SymbolRef model, Optional topic, Symbol de private void checkType(TypeDescriptor type, QualifiedName path) { switch (type.kind()) { case TEXT, BOOLEAN, NUMERIC, ENUMERATION, ALIGNMENT, FORMATTED, OID, - REFERENCE, STRUCTURE_VALUE, GENERIC, DEFERRED -> { } + BLACKBOX, CLASS_VALUE, ATTRIBUTE_PATH, REFERENCE, STRUCTURE_VALUE, GENERIC, DEFERRED -> { } case COLLECTION -> checkType(known(queries.type(typeRef(((CollectionType) type).elementType())), path), path); case DOMAIN_REFERENCE -> checkType(typeOf(ref(((DomainReferenceType) type).targetDomain()), path), path); case COORDINATE -> { @@ -247,9 +247,12 @@ private void checkGeometry(GeometryType geometry, QualifiedName path) { issues.add(new Issue(path, Feature.CUSTOM_LINE_FORM)); } } - if (geometry.lineAttributes().isPresent()) { - issues.add(new Issue(path, Feature.LINE_ATTRIBUTES)); - } + geometry.lineAttributes().ifPresent(id -> { + // The structure's effective fields are checked by projectObject, as for other structures. + if (!(known(queries.symbol(ref(id)), path) instanceof StructureSymbol)) { + issues.add(new Issue(path, Feature.LINE_ATTRIBUTES)); + } + }); geometry.vertexDomain().ifPresent(vertex -> checkType(typeOf(ref(vertex), path), path)); } diff --git a/interlis-transfer/src/main/java/guru/interlis/transfer/TransferValue.java b/interlis-transfer/src/main/java/guru/interlis/transfer/TransferValue.java index 38b09351..f2bc57da 100644 --- a/interlis-transfer/src/main/java/guru/interlis/transfer/TransferValue.java +++ b/interlis-transfer/src/main/java/guru/interlis/transfer/TransferValue.java @@ -1,11 +1,13 @@ package guru.interlis.transfer; import guru.interlis.model.CollectionKind; +import guru.interlis.model.QualifiedName; import guru.interlis.model.SnapshotMismatchException; import guru.interlis.model.SymbolRef; import java.math.BigDecimal; import java.math.BigInteger; +import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; @@ -17,7 +19,56 @@ * omitting its slot. Values and collection entries are nonnull. */ public sealed interface TransferValue permits TransferValue.Text, TransferValue.BooleanValue, TransferValue.Enumeration, TransferValue.NumberValue, TransferValue.Structured, - TransferValue.Repeated, TransferValue.Reference, TransferValue.Undefined, TransferGeometry { + TransferValue.Repeated, TransferValue.Reference, TransferValue.Binary, TransferValue.XmlFragment, + TransferValue.Oid, TransferValue.ClassName, TransferValue.AttributePath, + TransferValue.Undefined, TransferGeometry { + + /** Binary payload with content equality. The constructor and accessor copy the bytes. */ + record Binary(byte[] bytes) implements TransferValue { + public Binary { + bytes = bytes.clone(); + } + + @Override public byte[] bytes() { return bytes.clone(); } + /** Returns the payload length in bytes without copying it. */ + public int size() { return bytes.length; } + @Override public boolean equals(Object other) { + return other instanceof Binary binary && Arrays.equals(bytes, binary.bytes); + } + @Override public int hashCode() { return Arrays.hashCode(bytes); } + } + + /** Serialized XML fragment, possibly empty, with any necessary namespace declarations. + * Consumers must parse and validate it before emitting XML. */ + record XmlFragment(String xml) implements TransferValue { + public XmlFragment { + Objects.requireNonNull(xml, "xml"); + } + } + + /** Exact identifier spelling of an attribute value, including any identifier space prefix. */ + record Oid(String lexeme) implements TransferValue { + public Oid { + if (Objects.requireNonNull(lexeme, "lexeme").isBlank()) { + throw new IllegalArgumentException("An OID needs its lexical spelling"); + } + } + } + + /** Qualified declaration name in a CLASS or STRUCTURE slot. + * The slot's type determines the declaration family and any restrictions. */ + record ClassName(QualifiedName name) implements TransferValue { + public ClassName { + Objects.requireNonNull(name, "name"); + } + } + + /** Qualified owner name followed by its attribute name. */ + record AttributePath(QualifiedName name) implements TransferValue { + public AttributePath { + Objects.requireNonNull(name, "name"); + } + } /** Exact text, including the empty string. */ record Text(String value) implements TransferValue { diff --git a/interlis-transfer/src/test/java/guru/interlis/transfer/TransferPlannerUnsupportedFeatureTest.java b/interlis-transfer/src/test/java/guru/interlis/transfer/TransferPlannerUnsupportedFeatureTest.java deleted file mode 100644 index f6772e03..00000000 --- a/interlis-transfer/src/test/java/guru/interlis/transfer/TransferPlannerUnsupportedFeatureTest.java +++ /dev/null @@ -1,35 +0,0 @@ -package guru.interlis.transfer; - -import guru.interlis.model.QualifiedName; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static guru.interlis.transfer.TransferTestSchemas.compileInline; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertThrows; - -class TransferPlannerUnsupportedFeatureTest { - @Test - void rejectsAValidSchemaWithABlackboxSlotAtItsQualifiedPath() { - var schema = compileInline("Payload", """ - INTERLIS 2.4; - MODEL Payload AT "https://example.org" VERSION "1" = - DOMAIN Raw = BLACKBOX XML; - TOPIC Data = - CLASS Entry = - label : TEXT*20; - body : Raw; - END Entry; - END Data; - END Payload. - """); - var fingerprint = schema.fingerprint(); - var result = assertInstanceOf(TransferPlanningResult.Unsupported.class, TransferPlanner.plan(schema)); - assertEquals(List.of(new TransferPlanningResult.Issue(QualifiedName.of("Payload", "Data", "Entry", "body"), - TransferPlanningResult.Feature.VALUE_TYPE)), result.issues()); - assertThrows(UnsupportedOperationException.class, () -> result.issues().clear()); - assertEquals(fingerprint, schema.fingerprint()); - } -} diff --git a/interlis-transfer/src/test/java/guru/interlis/transfer/TransferPlannerValueTypesTest.java b/interlis-transfer/src/test/java/guru/interlis/transfer/TransferPlannerValueTypesTest.java new file mode 100644 index 00000000..4662540e --- /dev/null +++ b/interlis-transfer/src/test/java/guru/interlis/transfer/TransferPlannerValueTypesTest.java @@ -0,0 +1,45 @@ +package guru.interlis.transfer; + +import org.junit.jupiter.api.Test; + +import static guru.interlis.transfer.TransferTestSchemas.compileInline; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +class TransferPlannerValueTypesTest { + @Test + void plansBlackboxesOidPathsAndGeometryWithLineAttributesFromInterlis23() { + var schema = compileInline("FullValues", """ + INTERLIS 2.3; + MODEL FullValues (en) AT "https://example.org" VERSION "1" = + DOMAIN + Position = COORD 0.000 .. 200.000 [INTERLIS.m], 0.000 .. 200.000 [INTERLIS.m]; + TextId = OID TEXT*24; + NumberId = OID 0..1000; + OpenId = OID ANY; + TOPIC Data = + STRUCTURE Marks = + label : MANDATORY TEXT*20; + END Marks; + CLASS Entry = + document : BLACKBOX XML; + payload : BLACKBOX BINARY; + textId : TextId; + numberId : NumberId; + openId : OpenId; + uuid : INTERLIS.UUIDOID; + className : CLASS; + structureName : STRUCTURE; + attributeName : ATTRIBUTE; + trace : POLYLINE WITH (STRAIGHTS) VERTEX Position; + shape : SURFACE WITH (STRAIGHTS) VERTEX Position WITHOUT OVERLAPS > 0.001 LINE ATTRIBUTES Marks; + parcel : AREA WITH (STRAIGHTS) VERTEX Position WITHOUT OVERLAPS > 0.001 LINE ATTRIBUTES Marks; + END Entry; + END Data; + END FullValues. + """); + var fingerprint = schema.fingerprint(); + assertInstanceOf(TransferPlanningResult.Ready.class, TransferPlanner.plan(schema)); + assertEquals(fingerprint, schema.fingerprint()); + } +} diff --git a/interlis-transfer/src/test/java/guru/interlis/transfer/TransferValueTest.java b/interlis-transfer/src/test/java/guru/interlis/transfer/TransferValueTest.java index 57ca13ca..9c2427e7 100644 --- a/interlis-transfer/src/test/java/guru/interlis/transfer/TransferValueTest.java +++ b/interlis-transfer/src/test/java/guru/interlis/transfer/TransferValueTest.java @@ -1,6 +1,7 @@ package guru.interlis.transfer; import guru.interlis.model.CollectionKind; +import guru.interlis.model.QualifiedName; import guru.interlis.model.SnapshotId; import guru.interlis.model.SnapshotMismatchException; import guru.interlis.model.SymbolId; @@ -16,6 +17,7 @@ import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -25,6 +27,52 @@ class TransferValueTest { private static final SnapshotId SNAPSHOT = new SnapshotId("transfer-value-fixture"); + @Test + void retainsBinaryPayloadByValueAndCopiesBothDirections() { + byte[] bytes = {1, 2, -1}; + var value = new TransferValue.Binary(bytes); + bytes[0] = 0; + value.bytes()[1] = 0; + + assertArrayEquals(new byte[] {1, 2, -1}, value.bytes()); + var equal = new TransferValue.Binary(new byte[] {1, 2, -1}); + assertEquals(equal, value); + assertEquals(equal.hashCode(), value.hashCode()); + assertNotEquals(value, new TransferValue.Binary(new byte[0])); + } + + @Test + void keepsModelNamesAndOidsDistinctFromPlainText() { + var name = QualifiedName.of("Model", "Topic", "Object"); + assertNotEquals(new TransferValue.ClassName(name), new TransferValue.AttributePath(name)); + assertNotEquals(new TransferValue.Oid("domain:value"), new TransferValue.Text("domain:value")); + assertNotEquals(new TransferValue.XmlFragment(""), new TransferValue.Text("")); + } + + @Test + void checksOwnershipInsideEachGeometryContainerIncludingHoles() { + var type = ref("object"); + var field = ref("geometry"); + var identity = new TransferIdentity(new TransferIdentity.BasketId("basket", TransferIdentity.IdOrigin.SOURCE), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + var point = new TransferGeometry.Coordinate(List.of( + new TransferValue.NumberValue("0", BigDecimal.ZERO), + new TransferValue.NumberValue("0", BigDecimal.ZERO))); + var part = new TransferGeometry.CurvePart(point, List.of()); + var foreign = new SymbolRef(new SnapshotId("foreign-line-attributes"), new SymbolId("marks")); + var marked = new TransferGeometry.Polyline(List.of(part), + Optional.of(new TransferValue.Structured(foreign, Map.of()))); + var ring = new TransferGeometry.Ring(List.of(marked)); + var plain = new TransferGeometry.Ring(List.of(new TransferGeometry.Polyline(List.of(part)))); + var surface = new TransferGeometry.Surface(plain, List.of(ring)); + for (var geometry : List.of(marked, new TransferGeometry.MultiPolyline(List.of(marked)), + new TransferGeometry.Surface(ring, List.of()), surface, + new TransferGeometry.MultiSurface(List.of(surface)))) { + assertThrows(SnapshotMismatchException.class, () -> new TransferObject(identity, + new TransferValue.Structured(type, Map.of(field, geometry)))); + } + } + @Test void separatesMissingUndefinedAndPresentEmptyValues() { var type = ref("type"); diff --git a/interlis-xtf23/build.jenesis/test-inputs.properties b/interlis-xtf23/build.jenesis/test-inputs.properties new file mode 100644 index 00000000..8dc3d4ea --- /dev/null +++ b/interlis-xtf23/build.jenesis/test-inputs.properties @@ -0,0 +1,5 @@ +input.0=testdata/reference/iox/ili23/Reference23.ili +input.1=testdata/reference/iox/ili23/identity.xtf +input.2=testdata/reference/iox/ili23/unique-across-baskets.xtf +input.3=testdata/reference/iox/ili23/area-shared-edge.xtf +input.4=testdata/reference/iox/ili23/area-overlap.xtf diff --git a/interlis-xtf23/pom.xml b/interlis-xtf23/pom.xml new file mode 100644 index 00000000..37b300f5 --- /dev/null +++ b/interlis-xtf23/pom.xml @@ -0,0 +1,42 @@ + + + 4.0.0 + guru.interlisinterlis-java0.2.0-SNAPSHOT + interlis-xtf23 + + guru.interlisinterlis-transfer${project.version} + guru.interlisinterlis-model${project.version} + guru.interlisinterlis-compiler${project.version}test + org.junit.jupiterjunit-jupitertest + org.junit.platformjunit-platform-launchertest + + + diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Exception.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Exception.java new file mode 100644 index 00000000..e378ea64 --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Exception.java @@ -0,0 +1,35 @@ +package guru.interlis.xtf23; + +/** An XML, wire format, capability or I/O failure at the XTF 2.3 boundary. */ +public final class Xtf23Exception extends IllegalStateException { + /** Distinguishes format failures from an unsupported feature or host failure. */ + public enum Reason { + MALFORMED_XML, UNSAFE_XML, INVALID_WIRE, DUPLICATE_ID, UNSUPPORTED, LIMIT_EXCEEDED, IO, CONFIGURATION + } + + private final Reason reason; + private final int line; + private final int column; + + Xtf23Exception(Reason reason, String message, int line, int column, Throwable cause) { + super(line > 0 ? message + " (line " + line + ", column " + column + ")" : message, cause); + this.reason = reason; + this.line = line; + this.column = column; + } + + /** Returns the stable failure category. */ + public Reason reason() { + return reason; + } + + /** Returns the one-based XML line, or -1 when unavailable. */ + public int line() { + return line; + } + + /** Returns the one-based XML column, or -1 when unavailable. */ + public int column() { + return column; + } +} diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23IdentityScope.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23IdentityScope.java new file mode 100644 index 00000000..fd2937a3 --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23IdentityScope.java @@ -0,0 +1,24 @@ +package guru.interlis.xtf23; + +import guru.interlis.transfer.TransferIdentity.BasketId; +import guru.interlis.transfer.TransferIdentity.TransferId; + +import java.util.HashSet; +import java.util.Set; + +/** One FULL transfer's shared BID/TID space. The wire spelling is independent of IdOrigin. */ +final class Xtf23IdentityScope { + private Set ids = new HashSet<>(); + + boolean claim(BasketId id) { + return ids.add(id.value()); + } + + boolean claim(TransferId id) { + return ids.add(id.value()); + } + + void release() { + ids = Set.of(); + } +} diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Input.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Input.java new file mode 100644 index 00000000..0ac15778 --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Input.java @@ -0,0 +1,314 @@ +package guru.interlis.xtf23; + +import guru.interlis.execution.CancellationToken; +import guru.interlis.execution.OperationCancelledException; + +import javax.xml.XMLConstants; +import javax.xml.stream.Location; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayDeque; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +import static guru.interlis.xtf23.Xtf23Exception.Reason.*; +import static javax.xml.stream.XMLStreamConstants.*; + +/** Secure cursor over the XTF XML stream. */ +final class Xtf23Input { + static final int MAX_TEXT = 1_048_576; + static final int MAX_TOKEN_BYTES = 8 * MAX_TEXT; + static final int MAX_OBJECT_UNITS = 16 * MAX_TEXT; + private static final int MAX_OBJECT_TOKENS = 1_000_000; + static final int MAX_DEPTH = 128; + static final int MAX_ATTRIBUTES = 64; + private static final int MAX_NAME = 1024; + + private final BorrowedInput input; + private XMLStreamReader xml; + private int depth; + private final ArrayDeque namespaces = new ArrayDeque<>(); + private int objectTokens = -1; + private long objectUnits; + + Xtf23Input(InputStream input) { + this.input = new BorrowedInput(Objects.requireNonNull(input, "input")); + } + + void begin(CancellationToken cancellation) { + input.cancellation = cancellation; + } + + void finish() { + input.cancellation = CancellationToken.none(); + } + + void open() throws XMLStreamException { + var factory = XMLInputFactory.newDefaultFactory(); + try { + requireProperty(factory, XMLInputFactory.IS_NAMESPACE_AWARE, true); + requireProperty(factory, XMLInputFactory.IS_COALESCING, false); + requireProperty(factory, XMLInputFactory.IS_VALIDATING, false); + requireProperty(factory, XMLInputFactory.SUPPORT_DTD, false); + requireProperty(factory, XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + requireProperty(factory, XMLConstants.ACCESS_EXTERNAL_DTD, ""); + requireProperty(factory, "jdk.xml.maxElementDepth", MAX_DEPTH); + requireProperty(factory, "jdk.xml.elementAttributeLimit", MAX_ATTRIBUTES); + requireProperty(factory, "jdk.xml.maxXMLNameLimit", MAX_NAME); + } catch (IllegalArgumentException e) { + throw failure(CONFIGURATION, "Cannot configure the JDK XML input factory", null, e); + } + factory.setXMLResolver((publicId, systemId, baseUri, namespace) -> { + throw error(UNSAFE_XML, "External XML resolution is forbidden"); + }); + input.remaining = MAX_TOKEN_BYTES; + xml = factory.createXMLStreamReader(input); + if (!"1.0".equals(xml.getVersion()) || !"UTF-8".equalsIgnoreCase(xml.getCharacterEncodingScheme()) + || !"UTF-8".equalsIgnoreCase(xml.getEncoding())) { + throw error(INVALID_WIRE, "XTF 2.3 requires an XML 1.0 declaration and UTF-8 encoding"); + } + } + + private static void requireProperty(XMLInputFactory factory, String property, Object value) { + factory.setProperty(property, value); + if (!value.toString().equals(Objects.toString(factory.getProperty(property), null))) { + throw new IllegalArgumentException("JDK XML property did not take effect: " + property); + } + } + + XMLStreamReader cursor() { + return xml; + } + + void beginObject() { + objectTokens = MAX_OBJECT_TOKENS; + objectUnits = MAX_OBJECT_UNITS; + accountObjectToken(START_ELEMENT); + } + + void endObject() { + objectTokens = -1; + } + + private void accountObjectToken(int token) { + if (objectTokens < 0) { + return; + } + objectTokens--; + switch (token) { + case START_ELEMENT -> { + for (int i = 0; i < xml.getAttributeCount(); i++) { + objectUnits -= xml.getAttributeValue(i).length(); + } + for (int i = 0; i < xml.getNamespaceCount(); i++) { + objectUnits -= Objects.toString(xml.getNamespaceURI(i), "").length(); + } + } + case CHARACTERS, CDATA, SPACE, COMMENT -> objectUnits -= xml.getTextLength(); + case PROCESSING_INSTRUCTION -> objectUnits -= Objects.toString(xml.getPIData(), "").length(); + default -> { } + } + if (objectTokens < 0 || objectUnits < 0) { + throw error(LIMIT_EXCEEDED, "XTF object exceeds its XML event or character budget"); + } + } + + int next() throws XMLStreamException { + input.cancellation.throwIfCancellationRequested(); + input.remaining = MAX_TOKEN_BYTES; + if (xml.getEventType() == END_ELEMENT) { + if (!namespaces.isEmpty() && namespaces.peek().depth() == depth) { + namespaces.pop(); + } + depth--; + } + int token = xml.next(); + if (token == DTD || token == ENTITY_REFERENCE) { + throw error(UNSAFE_XML, "DTD declarations and custom XML entities are forbidden"); + } + if (token == START_ELEMENT) { + depth++; + if (xml.getNamespaceCount() > MAX_ATTRIBUTES) { + throw error(LIMIT_EXCEEDED, "Too many namespace declarations on an XTF element"); + } + if (xml.getNamespaceCount() > 0) { + var declarations = new TreeMap(); + for (int i = 0; i < xml.getNamespaceCount(); i++) { + String uri = Objects.toString(xml.getNamespaceURI(i), ""); + if (uri.length() > MAX_TEXT) { + throw error(LIMIT_EXCEEDED, "XTF namespace URI exceeds the text limit"); + } + declarations.put(Objects.toString(xml.getNamespacePrefix(i), ""), uri); + } + namespaces.push(new NamespaceScope(depth, declarations)); + } + } + accountObjectToken(token); + return token; + } + + Map fragmentNamespaces(boolean root) { + if (!root) { + return !namespaces.isEmpty() && namespaces.peek().depth() == depth + ? namespaces.peek().declarations() : Map.of(); + } + var bindings = new TreeMap(); + bindings.put("", ""); + for (var scope : namespaces.reversed()) { + bindings.putAll(scope.declarations()); + } + return bindings; + } + + private record NamespaceScope(int depth, Map declarations) { } + + int nextTag() throws XMLStreamException { + while (true) { + int token = next(); + switch (token) { + case START_ELEMENT, END_ELEMENT, END_DOCUMENT -> { return token; } + case COMMENT, PROCESSING_INSTRUCTION -> { } + case CHARACTERS, CDATA, SPACE -> { + if (!xml.isWhiteSpace()) { + throw error(INVALID_WIRE, "Unexpected text between XTF elements"); + } + } + default -> throw error(INVALID_WIRE, "Unexpected XML event " + token); + } + } + } + + String text() throws XMLStreamException { + var value = new StringBuilder(); + while (true) { + int token = next(); + switch (token) { + case END_ELEMENT -> { return value.toString(); } + case COMMENT, PROCESSING_INSTRUCTION -> { } + case CHARACTERS, CDATA, SPACE -> { + if (xml.getTextLength() > MAX_TEXT - value.length()) { + throw error(LIMIT_EXCEEDED, "XTF text value exceeds " + MAX_TEXT + " UTF-16 units"); + } + value.append(xml.getTextCharacters(), xml.getTextStart(), xml.getTextLength()); + } + default -> throw error(INVALID_WIRE, "Expected text without child elements"); + } + } + } + + void close() { + if (xml != null) { + try { + xml.close(); + } catch (XMLStreamException e) { + throw translate(e); + } finally { + xml = null; + namespaces.clear(); + } + } + } + + Xtf23Exception error(Xtf23Exception.Reason reason, String message) { + return failure(reason, message, xml == null ? null : xml.getLocation(), null); + } + + Xtf23Exception locate(Xtf23Exception error) { + if (xml == null || error.line() > 0) { + return error; + } + return failure(error.reason(), error.getMessage(), xml.getLocation(), error); + } + + Xtf23Exception translate(XMLStreamException e) { + Throwable nested = e.getNestedException(); + if (nested instanceof Xtf23Exception failure) { + return failure; + } + if (nested instanceof OperationCancelledException cancelled) { + throw cancelled; + } + String message = e.getMessage(); + if (nested instanceof TokenLimit || message != null && message.contains("JAXP0001")) { + return failure(LIMIT_EXCEEDED, "XML structure or token exceeds the XTF reader limits", e.getLocation(), e); + } + if (nested instanceof SourceFailure) { + return failure(IO, "Cannot read the XTF input stream", e.getLocation(), e); + } + return failure(MALFORMED_XML, "Malformed XML: " + message, e.getLocation(), e); + } + + private static Xtf23Exception failure(Xtf23Exception.Reason reason, String message, + Location location, Throwable cause) { + return new Xtf23Exception(reason, message, location == null ? -1 : location.getLineNumber(), + location == null ? -1 : location.getColumnNumber(), cause); + } + + private static final class TokenLimit extends IOException { } + + private static final class SourceFailure extends IOException { + SourceFailure(IOException cause) { + super(cause); + } + } + + /** StAX can close the source at EOF. Limit parser reads within an individual XML advance. */ + private static final class BorrowedInput extends FilterInputStream { + private CancellationToken cancellation = CancellationToken.none(); + private int remaining; + + BorrowedInput(InputStream input) { + super(input); + } + + private void requireBudget() throws TokenLimit { + cancellation.throwIfCancellationRequested(); + if (remaining == 0) { + throw new TokenLimit(); + } + } + + @Override + public int read() throws IOException { + requireBudget(); + int value; + try { + value = in.read(); + } catch (IOException failure) { + throw new SourceFailure(failure); + } + if (value != -1) { + remaining--; + } + return value; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + if (length == 0) { + return 0; + } + requireBudget(); + int count; + try { + count = in.read(buffer, offset, Math.min(length, remaining)); + } catch (IOException failure) { + throw new SourceFailure(failure); + } + if (count > 0) { + remaining -= count; + } + return count; + } + + @Override + public void close() { + // The caller owns the source even when StAX reaches EOF or abandons parsing. + } + } +} diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Layout.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Layout.java new file mode 100644 index 00000000..d05d50a5 --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Layout.java @@ -0,0 +1,226 @@ +package guru.interlis.xtf23; + +import guru.interlis.model.CardinalityUpper; +import guru.interlis.model.CoordinateType; +import guru.interlis.model.DomainReferenceType; +import guru.interlis.model.DomainSymbol; +import guru.interlis.model.GeometryType; +import guru.interlis.model.Modifier; +import guru.interlis.model.Origin; +import guru.interlis.model.SymbolId; +import guru.interlis.model.SymbolKind; +import guru.interlis.model.SymbolRef; +import guru.interlis.model.TypeDescriptor; +import guru.interlis.model.TypeRef; +import guru.interlis.transfer.AttributeSlot; +import guru.interlis.transfer.ObjectDescriptor; +import guru.interlis.transfer.RoleSlot; +import guru.interlis.transfer.TransferPlan; +import guru.interlis.transfer.TransferSlot; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** Derives the XTF 2.3 wire order and association placement from the neutral plan. */ +final class Xtf23Layout { + record Embedded(ObjectDescriptor association, RoleSlot owner, RoleSlot opposite, RoleSlot rootOpposite) { } + + record Field(TransferSlot slot, List embeddings, int level) { + String name() { + return embeddings.isEmpty() ? slot.name().declared().last() : embeddings.getFirst().opposite().name().declared().last(); + } + + private int group() { + return embeddings.isEmpty() ? (slot instanceof RoleSlot ? 0 : 1) : 2; + } + } + + private static final Comparator WIRE_ORDER = Comparator.comparingInt(Field::level) + .thenComparingInt(Field::group) + .thenComparing(field -> field.embeddings().isEmpty() ? "" : field.name()) + .thenComparingInt(field -> field.slot().member().position()); + + private final TransferPlan plan; + private final Map embedded; + + Xtf23Layout(TransferPlan plan) { + this.plan = plan; + var result = new HashMap(); + for (var descriptor : plan.objects()) { + if (descriptor.kind() == SymbolKind.ASSOCIATION && descriptor.transferable()) { + placement(descriptor).ifPresent(value -> result.put(descriptor.symbol(), value)); + } + } + embedded = Map.copyOf(result); + } + + Optional embedded(ObjectDescriptor descriptor) { + return Optional.ofNullable(embedded.get(descriptor.symbol())); + } + + boolean identified(ObjectDescriptor association) { + if (!association.oidRules().isEmpty()) { + return association.oidRules().stream().anyMatch(rule -> !rule.basket() && rule.domain().isPresent()); + } + return ancestors(association).stream().anyMatch(descriptor -> descriptor.modifiers().contains(Modifier.OID)); + } + + List fields(ObjectDescriptor descriptor, Set activeTypes, boolean payloadOnly) { + var inheritance = ancestors(descriptor); + var levels = new HashMap(); + for (int i = 0; i < inheritance.size(); i++) { + for (var slot : inheritance.get(i).slots()) { + levels.putIfAbsent(slot.member().position(), i); + } + } + var fields = new ArrayList(); + for (var slot : descriptor.slots()) { + if (slot instanceof AttributeSlot attribute && !attribute.attribute().transientAttribute() + || slot instanceof RoleSlot && !payloadOnly) { + fields.add(new Field(slot, List.of(), levels.get(slot.member().position()))); + } + } + if (!payloadOnly) { + addEmbedded(fields, descriptor, activeTypes, inheritance); + } + fields.sort(WIRE_ORDER); + return List.copyOf(fields); + } + + private void addEmbedded(List fields, ObjectDescriptor descriptor, Set activeTypes, + List inheritance) { + if (descriptor.kind() == SymbolKind.STRUCTURE) { + return; + } + var byName = new HashMap>(); + for (var entry : embedded.entrySet()) { + var value = entry.getValue(); + if (activeTypes.contains(entry.getKey()) && matchesRoots(descriptor, value.owner().role().targetRoots())) { + byName.computeIfAbsent(value.opposite().name().declared().last(), unused -> new ArrayList<>()).add(value); + } + } + for (var variants : byName.values()) { + // Resolve concrete variants in a stable order even if the plan was indexed in a hash map. + variants.sort(Comparator.comparing(value -> value.association().name().declared().toString())); + var rootRole = variants.getFirst().rootOpposite(); + int level = inheritance.size() - 1; + for (var variant : variants) { + var targets = variant.owner().role().targets().stream().map(target -> target.base()).toList(); + for (int i = 0; i < level; i++) { + if (targets.contains(inheritance.get(i).symbol().symbol())) { + level = i; + break; + } + } + } + fields.add(new Field(rootRole, List.copyOf(variants), level)); + } + } + + boolean matchesRoots(ObjectDescriptor descriptor, List allowed) { + return allowed.isEmpty() || ancestors(descriptor).stream() + .anyMatch(base -> allowed.contains(base.symbol().symbol())); + } + + TypeDescriptor type(TypeRef ref) { + var descriptor = plan.type(ref).known().orElseThrow(); + while (descriptor instanceof DomainReferenceType alias) { + var domain = (DomainSymbol) plan.symbol(new SymbolRef(plan.snapshotId(), alias.targetDomain())) + .known().orElseThrow(); + descriptor = plan.type(new TypeRef(plan.snapshotId(), domain.typeId())).known().orElseThrow(); + } + return descriptor; + } + + int dimensions(CoordinateType type) { + if (type.multiCoordinate() || type.dimensions() < 2) { + throw unsupported("Only single two- and three-dimensional XTF coordinates are supported"); + } + return type.dimensions(); + } + + record CurveEncoding(int dimensions, boolean straights, boolean arcs, List lineAttributeRoots) { } + + CurveEncoding curveEncoding(GeometryType type) { + var domain = type.vertexDomain().flatMap(id -> plan.symbol(new SymbolRef(plan.snapshotId(), id)).known()); + if (domain.isEmpty() || !(domain.orElseThrow() instanceof DomainSymbol vertex) + || !(type(new TypeRef(plan.snapshotId(), vertex.typeId())) instanceof CoordinateType coordinate)) { + throw unsupported("XTF line type requires a concrete coordinate domain"); + } + boolean straights = type.lineForms().isEmpty(); + boolean arcs = type.lineForms().isEmpty(); + for (var id : type.lineForms()) { + var symbol = plan.symbol(new SymbolRef(plan.snapshotId(), id)).known().orElseThrow(); + if (!(symbol.origin() instanceof Origin.PredefinedOrigin origin)) { + throw unsupported("Custom XTF line forms are unsupported"); + } + switch (origin.key()) { + case "INTERLIS.STRAIGHTS" -> straights = true; + case "INTERLIS.ARCS" -> arcs = true; + default -> throw unsupported("Unsupported XTF line form " + origin.key()); + } + } + return new CurveEncoding(dimensions(coordinate), straights, arcs, type.lineAttributes().stream().toList()); + } + + private static Xtf23Exception unsupported(String message) { + return new Xtf23Exception(Xtf23Exception.Reason.UNSUPPORTED, message, -1, -1, null); + } + + private Optional placement(ObjectDescriptor association) { + if (identified(association)) { + return Optional.empty(); + } + var root = ancestors(association).getFirst(); + var rootRoles = roles(root); + var effective = roles(association); + if (rootRoles.size() != 2 || effective.size() != 2) { + return Optional.empty(); + } + boolean firstMany = many(rootRoles.get(0)); + boolean secondMany = many(rootRoles.get(1)); + if (firstMany && secondMany) { + return Optional.empty(); + } + int owner = firstMany ? 0 : 1; + if (!local(root, rootRoles.get(owner))) { + if (firstMany || secondMany || !local(root, rootRoles.getFirst())) { + return Optional.empty(); + } + owner = 0; + } + int opposite = 1 - owner; + return Optional.of(new Embedded(association, effective.get(owner), effective.get(opposite), rootRoles.get(opposite))); + } + + private boolean local(ObjectDescriptor association, RoleSlot role) { + return role.role().targets().stream().allMatch(target -> + plan.object(new SymbolRef(plan.snapshotId(), target.base())) + .filter(object -> object.topic().equals(association.topic())).isPresent()); + } + + private static boolean many(RoleSlot role) { + return !(role.role().cardinality().upper() instanceof CardinalityUpper.Finite finite) + || finite.value().compareTo(BigInteger.ONE) > 0; + } + + private static List roles(ObjectDescriptor association) { + return association.slots().stream().filter(RoleSlot.class::isInstance).map(RoleSlot.class::cast).toList(); + } + + private List ancestors(ObjectDescriptor descriptor) { + var hierarchy = new ArrayList(); + hierarchy.add(descriptor); + while (descriptor.baseType().isPresent()) { + descriptor = plan.object(descriptor.baseType().orElseThrow()).orElseThrow(); + hierarchy.add(descriptor); + } + return hierarchy.reversed(); + } +} diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Lexemes.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Lexemes.java new file mode 100644 index 00000000..deadcba7 --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Lexemes.java @@ -0,0 +1,61 @@ +package guru.interlis.xtf23; + +import guru.interlis.model.OidType; +import guru.interlis.transfer.TransferValue.NumberValue; + +import java.math.BigDecimal; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Pattern; + +/** Lexical rules for XTF values, independent of model constraints. */ +final class Xtf23Lexemes { + private static final Pattern DECIMAL = + Pattern.compile("[+-]?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?(?:[eE][+-]?[0-9]+)?"); + private static final Pattern NAME = Pattern.compile("[A-Za-z][A-Za-z0-9_]*"); + private static final Pattern STRUCTURED_NUMBER = Pattern.compile("[+-]?[0-9]++(?::[0-9]++)++(?:\\.[0-9]++)?"); + static final int MAX_NUMERIC_LENGTH = 4096; + static final int MAX_MODEL_PATH_LENGTH = 4096; + static final Pattern POSITION = Pattern.compile("[0-9]+"); + + private Xtf23Lexemes() { } + + static boolean singleLine(String value) { + return value.indexOf('\r') < 0 && value.indexOf('\n') < 0 && value.indexOf('\t') < 0; + } + + static boolean enumeration(String value) { + return value.length() <= 4096 && Arrays.stream(value.split("\\.", -1)) + .allMatch(part -> !part.equals("OTHERS") && NAME.matcher(part).matches()); + } + + static boolean decimal(String value) { + return DECIMAL.matcher(value).matches(); + } + + static boolean oidValue(OidType type, String identifier) { + int separator = identifier.indexOf(':'); + String local = identifier.substring(separator + 1); + return switch (type.oidKind()) { + case ANY -> separator >= 0; + case NUMERIC -> decimal(local); + case TEXT -> local.charAt(0) < '0' || local.charAt(0) > '9'; + case UUID -> true; + }; + } + + static boolean modelPath(List segments, boolean attribute) { + int minimum = attribute ? 3 : 2; + return segments.size() >= minimum && segments.size() <= minimum + 1 + && segments.stream().allMatch(part -> NAME.matcher(part).matches()); + } + + static NumberValue number(String value) { + return new NumberValue(value, new BigDecimal(value)); + } + + static boolean nonDecimalNumber(String value) { + return value.equals("PI") || value.equals("LNBASE") + || (value.indexOf(':') >= 0 && STRUCTURED_NUMBER.matcher(value).matches()); + } +} diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23OidSpace.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23OidSpace.java new file mode 100644 index 00000000..2ea8971d --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23OidSpace.java @@ -0,0 +1,25 @@ +package guru.interlis.xtf23; + +import java.util.Objects; +import java.util.regex.Pattern; + +/** + * One XTF 2.3 header declaration connecting an XML-ID prefix to its OID domain. + * Both components retain their exact source spelling. + */ +public record Xtf23OidSpace(String alias, String domain) { + static final String NC_NAME = "[A-Za-z_][A-Za-z0-9_.-]*"; + private static final Pattern ALIAS = Pattern.compile(NC_NAME); + private static final Pattern QUALIFIED_DOMAIN = + Pattern.compile("[A-Za-z][A-Za-z0-9_]*(?:\\.[A-Za-z][A-Za-z0-9_]*)+"); + + /** Creates a declaration with a nonnull XML prefix and qualified INTERLIS domain name. */ + public Xtf23OidSpace { + Objects.requireNonNull(alias, "alias"); + Objects.requireNonNull(domain, "domain"); + if (alias.length() > 1024 || domain.length() > 1024 || !ALIAS.matcher(alias).matches() + || !QUALIFIED_DOMAIN.matcher(domain).matches()) { + throw new IllegalArgumentException("Invalid OIDSPACE alias or qualified domain name"); + } + } +} diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Output.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Output.java new file mode 100644 index 00000000..b9bac358 --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Output.java @@ -0,0 +1,166 @@ +package guru.interlis.xtf23; + +import guru.interlis.execution.CancellationToken; +import guru.interlis.execution.OperationCancelledException; + +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamWriter; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +import static guru.interlis.xtf23.Xtf23Exception.Reason.*; + +final class Xtf23Output { + private final BorrowedOutput output; + private XMLStreamWriter xml; + private int depth; + + Xtf23Output(OutputStream output) { + this.output = new BorrowedOutput(Objects.requireNonNull(output, "output")); + } + + void begin(CancellationToken cancellation) { + output.cancellation = cancellation; + } + + void finish() { + output.cancellation = CancellationToken.none(); + } + + void open() throws XMLStreamException { + xml = XMLOutputFactory.newDefaultFactory().createXMLStreamWriter(output, "UTF-8"); + xml.writeStartDocument("UTF-8", "1.0"); + start("TRANSFER"); + xml.writeDefaultNamespace(Xtf23Reader.NAMESPACE); + } + + void start(String name) throws XMLStreamException { + output.cancellation.throwIfCancellationRequested(); + if (depth == Xtf23Input.MAX_DEPTH) { + throw error(LIMIT_EXCEEDED, "XTF output exceeds the supported XML depth"); + } + xml.writeStartElement(name); + depth++; + } + + void end() throws XMLStreamException { + output.cancellation.throwIfCancellationRequested(); + xml.writeEndElement(); + depth--; + } + + void attribute(String name, String value) throws XMLStreamException { + if (!Xtf23Lexemes.singleLine(value)) { + throw error(INVALID_WIRE, "XTF attribute " + name + " must be single-line text"); + } + checkCharacters(value); + xml.writeAttribute(name, value); + } + + void text(String value) throws XMLStreamException { + checkCharacters(value); + int start = 0; + for (int carriage = value.indexOf('\r'); carriage >= 0; carriage = value.indexOf('\r', start)) { + xml.writeCharacters(value.substring(start, carriage)); + xml.writeEntityRef("#xD"); + start = carriage + 1; + } + xml.writeCharacters(value.substring(start)); + } + + void fragment(String value) throws XMLStreamException { + checkCharacters(value); + String checked = Xtf23XmlFragment.validate(value, depth, output.cancellation); + // Finish the pending wrapper start tag before inserting checked XML events serialized as UTF-8. + xml.writeCharacters(""); + xml.flush(); + try { + output.write(checked.getBytes(StandardCharsets.UTF_8)); + } catch (IOException failure) { + throw new XMLStreamException(failure); + } + } + + void endDocument() throws XMLStreamException { + xml.writeEndDocument(); + xml.flush(); + } + + void close() { + if (xml != null) { + try { + xml.close(); + } catch (XMLStreamException failure) { + throw translate(failure); + } finally { + xml = null; + } + } + } + + private void checkCharacters(String value) { + output.cancellation.throwIfCancellationRequested(); + if (value.length() > Xtf23Input.MAX_TEXT) { + throw error(LIMIT_EXCEEDED, "XTF output text exceeds the scalar text limit"); + } + if (!value.codePoints().allMatch(Xtf23Output::xmlCharacter)) { + throw error(INVALID_WIRE, "Text contains a character outside XML 1.0"); + } + } + + private static boolean xmlCharacter(int code) { + return code == 9 || code == 10 || code == 13 || (code >= 0x20 && code <= 0xd7ff) + || (code >= 0xe000 && code <= 0xfffd) || (code >= 0x10000 && code <= 0x10ffff); + } + + static Xtf23Exception translate(XMLStreamException failure) { + var cause = failure.getNestedException(); + if (cause instanceof OperationCancelledException cancelled) { + throw cancelled; + } + if (cause instanceof IOException) { + return new Xtf23Exception(IO, "Cannot write the XTF output stream", -1, -1, failure); + } + return new Xtf23Exception(CONFIGURATION, "Cannot produce XTF XML: " + failure.getMessage(), + -1, -1, failure); + } + + static Xtf23Exception error(Xtf23Exception.Reason reason, String message) { + return new Xtf23Exception(reason, message, -1, -1, null); + } + + private static final class BorrowedOutput extends FilterOutputStream { + private CancellationToken cancellation = CancellationToken.none(); + + BorrowedOutput(OutputStream output) { + super(output); + } + + @Override + public void write(int value) throws IOException { + cancellation.throwIfCancellationRequested(); + out.write(value); + } + + @Override + public void write(byte[] values, int offset, int length) throws IOException { + cancellation.throwIfCancellationRequested(); + out.write(values, offset, length); + } + + @Override + public void flush() throws IOException { + cancellation.throwIfCancellationRequested(); + out.flush(); + } + + @Override + public void close() { + // The caller owns the stream on completion, cancellation and failure. + } + } +} diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Reader.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Reader.java new file mode 100644 index 00000000..0ad22c71 --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Reader.java @@ -0,0 +1,861 @@ +package guru.interlis.xtf23; + +import guru.interlis.execution.CancellationToken; +import guru.interlis.model.BlackboxType; +import guru.interlis.model.CollectionType; +import guru.interlis.model.CoordinateType; +import guru.interlis.model.GeometryType; +import guru.interlis.model.LanguageVersion; +import guru.interlis.model.ModelSymbol; +import guru.interlis.model.Modifier; +import guru.interlis.model.OidType; +import guru.interlis.model.QualifiedName; +import guru.interlis.model.ReferenceType; +import guru.interlis.model.StructureValueType; +import guru.interlis.model.SymbolId; +import guru.interlis.model.SymbolKind; +import guru.interlis.model.SymbolRef; +import guru.interlis.model.TextKind; +import guru.interlis.model.TextType; +import guru.interlis.model.TypeDescriptor; +import guru.interlis.model.TypeRef; +import guru.interlis.source.SourceId; +import guru.interlis.transfer.AttributeSlot; +import guru.interlis.transfer.ModelDescriptor; +import guru.interlis.transfer.ObjectDescriptor; +import guru.interlis.transfer.RoleSlot; +import guru.interlis.transfer.TopicDescriptor; +import guru.interlis.transfer.TransferEvent; +import guru.interlis.transfer.TransferGeometry; +import guru.interlis.transfer.TransferIdentity; +import guru.interlis.transfer.TransferName; +import guru.interlis.transfer.TransferObject; +import guru.interlis.transfer.TransferPlan; +import guru.interlis.transfer.TransferReader; +import guru.interlis.transfer.TransferValue; + +import javax.xml.XMLConstants; +import javax.xml.stream.XMLStreamException; +import java.io.InputStream; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static guru.interlis.transfer.TransferEvent.*; +import static guru.interlis.transfer.TransferIdentity.*; +import static guru.interlis.xtf23.Xtf23Exception.Reason.*; +import static javax.xml.stream.XMLStreamConstants.*; + +/** + * Pulls one XTF 2.3 FULL transfer from a caller-owned byte stream. Constructing this reader + * performs no input. Close releases the parser without closing the supplied stream. + * Events are immutable and can be retained after the reader advances. + * The XML reader bounds depth to 128 elements, scalar text to 1,048,576 UTF-16 + * code units, and input read during any individual parser advance to 8 MiB. + * One object, including its nested values, is limited to 1,000,000 XML events and + * 16,777,216 UTF-16 code units of XML text and attribute values. + * The reader holds a transfer-wide index of occurrence IDs whose size grows with + * the number of distinct BIDs and TIDs. + */ +public final class Xtf23Reader extends TransferReader { + static final String NAMESPACE = "http://www.interlis.ch/INTERLIS2.3"; + static final Pattern XML_ID = Pattern.compile("(?:" + Xtf23OidSpace.NC_NAME + ":)?[A-Za-z0-9_][A-Za-z0-9_.-]*"); + private static final String VERSION = "2.3"; + static final int MAX_OID_SPACES = 4096; + static final int MAX_IDENTIFIER = 4096; + + private final TransferPlan plan; + private final Xtf23Input xml; + private final Optional source; + private final Map models; + private final Map topics; + private final Map objects; + private final Xtf23Layout layout; + private final Xtf23IdentityScope identities = new Xtf23IdentityScope(); + private final Map> fieldIndexes = new HashMap<>(); + private Map oidSpaces = Map.of(); + private Set activeTypes = Set.of(); + private List oidSpaceDeclarations = List.of(); + private State state = State.BEFORE_TRANSFER; + private BasketId basket; + + /** Creates a reader over the nonnull plan and caller-owned input. */ + public Xtf23Reader(TransferPlan plan, InputStream input) { + this(plan, input, Optional.empty()); + } + + /** Creates a reader with an optional nonnull opaque source identity for events. */ + public Xtf23Reader(TransferPlan plan, InputStream input, Optional source) { + super(plan); + this.plan = plan; + this.source = Objects.requireNonNull(source, "source"); + xml = new Xtf23Input(input); + models = index(plan.models(), ModelDescriptor::name); + topics = index(plan.topics(), TopicDescriptor::name); + objects = index(plan.objects(), ObjectDescriptor::name); + layout = new Xtf23Layout(plan); + } + + /** Returns the immutable, ordered OIDSPACE declarations after reading the header. + * An empty list represents no declarations, including before the first read. */ + public List oidSpaces() { + return oidSpaceDeclarations; + } + + private static Map index(List descriptors, Function names) { + return descriptors.stream().collect(Collectors.toUnmodifiableMap( + value -> names.apply(value).declared().toString(), Function.identity())); + } + + @Override + protected Optional readEvent(CancellationToken cancellation) { + xml.begin(cancellation); + try { + return switch (state) { + case BEFORE_TRANSFER -> Optional.of(startTransfer()); + case BETWEEN_BASKETS -> Optional.of(basketOrEndTransfer()); + case IN_BASKET -> Optional.of(objectOrEndBasket()); + case DONE -> Optional.empty(); + }; + } catch (XMLStreamException failure) { + clearTransferState(); + throw xml.translate(failure); + } catch (RuntimeException failure) { + clearTransferState(); + throw failure instanceof Xtf23Exception xtfFailure ? xml.locate(xtfFailure) : failure; + } finally { + xml.finish(); + } + } + + @Override + protected void release() { + clearTransferState(); + xml.close(); + } + + private void clearTransferState() { + basket = null; + oidSpaces = Map.of(); + identities.release(); + clearBasketState(); + } + + private void clearBasketState() { + activeTypes = Set.of(); + fieldIndexes.clear(); + } + + private StartTransfer startTransfer() throws XMLStreamException { + xml.open(); + require(START_ELEMENT, xml.nextTag(), "TRANSFER"); + attributes(true); + require(START_ELEMENT, xml.nextTag(), "HEADERSECTION"); + var header = attributes(false, "VERSION", "SENDER"); + if (!VERSION.equals(required(header, "VERSION"))) { + throw xml.error(INVALID_WIRE, "Expected XTF header VERSION " + VERSION); + } + String sender = required(header, "SENDER"); + require(START_ELEMENT, xml.nextTag(), "MODELS"); + attributes(false); + var declarations = readModels(); + + int token = xml.nextTag(); + if (is(START_ELEMENT, token, "ALIAS")) { + attributes(false); + if (!is(END_ELEMENT, xml.nextTag(), "ALIAS")) { + throw xml.error(UNSUPPORTED, "Nonempty polymorphic ALIAS tables are unsupported"); + } + token = xml.nextTag(); + } + Map spaces = Map.of(); + if (is(START_ELEMENT, token, "OIDSPACES")) { + spaces = readOidSpaces(); + token = xml.nextTag(); + } + Optional comment = Optional.empty(); + if (is(START_ELEMENT, token, "COMMENT")) { + attributes(false); + comment = Optional.of(xml.text()); + token = xml.nextTag(); + } + require(END_ELEMENT, token, "HEADERSECTION"); + require(START_ELEMENT, xml.nextTag(), "DATASECTION"); + attributes(false); + oidSpaces = spaces; + oidSpaceDeclarations = List.copyOf(spaces.values()); + state = State.BETWEEN_BASKETS; + return new StartTransfer(source, Optional.of(sender), comment, declarations); + } + + private List readModels() throws XMLStreamException { + var declarations = new ArrayList(); + var declared = new HashSet(); + int token = xml.nextTag(); + while (!is(END_ELEMENT, token, "MODELS")) { + require(START_ELEMENT, token, "MODEL"); + var attributes = attributes(false, "NAME", "VERSION", "URI"); + String name = required(attributes, "NAME"); + String version = required(attributes, "VERSION"); + String issuer = required(attributes, "URI"); + var model = models.get(name); + if (model == null) { + throw xml.error(INVALID_WIRE, "Unknown XTF model " + name); + } + if (model.languageVersion() != LanguageVersion.ILI23) { + throw xml.error(UNSUPPORTED, "Model " + name + " does not use INTERLIS 2.3"); + } + var symbol = (ModelSymbol) plan.symbol(model.symbol()).known().orElseThrow(); + if (symbol.translation().isPresent()) { + throw xml.error(UNSUPPORTED, "Translated models require an XTF ALIAS table"); + } + if (!version.equals(model.modelVersion()) || !issuer.equals(symbol.issuer())) { + throw xml.error(INVALID_WIRE, "XTF declaration conflicts with model " + name); + } + if (!declared.add(model.symbol())) { + throw xml.error(INVALID_WIRE, "Repeated XTF model declaration " + name); + } + require(END_ELEMENT, xml.nextTag(), "MODEL"); + declarations.add(new ModelDeclaration(model.symbol(), model.languageVersion(), + Optional.of(version), Optional.of(issuer))); + token = xml.nextTag(); + } + if (declarations.isEmpty()) { + throw xml.error(INVALID_WIRE, "MODELS requires at least one MODEL"); + } + return declarations; + } + + private Map readOidSpaces() throws XMLStreamException { + attributes(false); + var spaces = new LinkedHashMap(); + int token = xml.nextTag(); + while (!is(END_ELEMENT, token, "OIDSPACES")) { + if (spaces.size() == MAX_OID_SPACES) { + throw xml.error(LIMIT_EXCEEDED, "Too many XTF OIDSPACE declarations"); + } + require(START_ELEMENT, token, "OIDSPACE"); + var attributes = attributes(false, "NAME", "OIDDOMAIN"); + String alias = required(attributes, "NAME"); + String domain = required(attributes, "OIDDOMAIN"); + Xtf23OidSpace space; + try { + space = new Xtf23OidSpace(alias, domain); + } catch (IllegalArgumentException e) { + throw xml.error(INVALID_WIRE, e.getMessage()); + } + if (spaces.putIfAbsent(alias, space) != null) { + throw xml.error(INVALID_WIRE, "Repeated OIDSPACE alias " + alias); + } + require(END_ELEMENT, xml.nextTag(), "OIDSPACE"); + token = xml.nextTag(); + } + if (spaces.isEmpty()) { + throw xml.error(INVALID_WIRE, "OIDSPACES requires at least one OIDSPACE"); + } + return spaces; + } + + private TransferEvent basketOrEndTransfer() throws XMLStreamException { + int token = xml.nextTag(); + if (is(END_ELEMENT, token, "DATASECTION")) { + require(END_ELEMENT, xml.nextTag(), "TRANSFER"); + if (xml.nextTag() != END_DOCUMENT) { + throw xml.error(INVALID_WIRE, "Unexpected XML after the XTF transfer"); + } + state = State.DONE; + clearTransferState(); + return new EndTransfer(); + } + String tag = elementName(token); + var topic = topic(tag); + var attributes = attributes(false, "BID", "TOPICS", "KIND", "STARTSTATE", "ENDSTATE", "CONSISTENCY"); + switch (attributes.getOrDefault("KIND", "FULL")) { + case "FULL" -> { } + case "INITIAL", "UPDATE" -> throw xml.error(UNSUPPORTED, "Incremental XTF baskets are unsupported"); + default -> throw xml.error(INVALID_WIRE, "Unknown XTF basket KIND"); + } + if (attributes.containsKey("STARTSTATE") || attributes.containsKey("ENDSTATE")) { + throw xml.error(INVALID_WIRE, "A FULL basket cannot carry incremental state"); + } + var id = basketId(required(attributes, "BID")); + var extensions = new ArrayList(); + if (attributes.containsKey("TOPICS")) { + var seen = new HashSet(); + for (String name : attributes.get("TOPICS").split(",", -1)) { + SymbolRef extension = topic(name).symbol(); + if (!seen.add(extension)) { + throw xml.error(INVALID_WIRE, "Repeated XTF extension topic " + name); + } + extensions.add(extension); + } + } + var consistency = consistency(attributes); + if (!identities.claim(id)) { + throw xml.error(DUPLICATE_ID, "XTF FULL BID/TID collision for " + id.value()); + } + var allowed = new HashSet<>(topic.objectTypes()); + for (var extension : extensions) { + allowed.addAll(plan.topic(extension).orElseThrow().objectTypes()); + } + activeTypes = Set.copyOf(allowed); + basket = id; + state = State.IN_BASKET; + return new StartBasket(id, topic.symbol(), extensions, BasketKind.FULL, consistency, + Optional.empty(), Optional.empty(), stableId(id.value()), List.of()); + } + + private TransferEvent objectOrEndBasket() throws XMLStreamException { + int token = xml.nextTag(); + if (token == END_ELEMENT) { + var event = new EndBasket(basket); + basket = null; + clearBasketState(); + state = State.BETWEEN_BASKETS; + return event; + } + String tag = elementName(token); + var descriptor = objects.get(tag); + if (descriptor == null || !descriptor.transferable()) { + throw xml.error(INVALID_WIRE, "Unknown or nontransferable XTF object tag " + tag); + } + if (layout.embedded(descriptor).isPresent()) { + throw xml.error(INVALID_WIRE, "Embedded XTF association cannot occur as a standalone object: " + tag); + } + try { + xml.beginObject(); + return readObject(descriptor, tag); + } finally { + xml.endObject(); + } + } + + private ObjectEvent readObject(ObjectDescriptor descriptor, String tag) throws XMLStreamException { + var attributes = attributes(false, "TID", "BID", "OPERATION", "CONSISTENCY"); + if (attributes.containsKey("OPERATION")) { + throw xml.error(INVALID_WIRE, "A FULL object cannot carry an incremental OPERATION"); + } + boolean requiresTid = descriptor.kind() == SymbolKind.CLASS || layout.identified(descriptor); + if (requiresTid != attributes.containsKey("TID")) { + throw xml.error(INVALID_WIRE, requiresTid ? "Missing XTF attribute TID" + : "A link without its own identification cannot carry TID"); + } + var tid = Optional.ofNullable(attributes.get("TID")) + .map(value -> new TransferId(identifier(value), IdOrigin.SOURCE)); + tid.ifPresent(value -> { + if (!identities.claim(value)) { + throw xml.error(DUPLICATE_ID, "XTF FULL BID/TID collision for " + value.value()); + } + }); + var originalBasket = Optional.ofNullable(attributes.get("BID")).map(this::basketId); + var consistency = consistency(attributes); + var identity = new TransferIdentity(basket, originalBasket, tid, + tid.flatMap(value -> stableId(value.value())).map(ObjectOid::new), source); + var data = readFields(descriptor, tag, false); + return new ObjectEvent(new TransferObject(identity, + new TransferValue.Structured(descriptor.symbol(), data)), Optional.empty(), consistency); + } + + private record PositionedField(Xtf23Layout.Field field, int position) { } + + private static Map indexFields(List fields) { + var index = new HashMap(); + for (int i = 0; i < fields.size(); i++) { + var field = fields.get(i); + if (index.putIfAbsent(field.name(), new PositionedField(field, i)) != null) { + index.put(field.name(), new PositionedField(field, -1)); + } + } + return index; + } + + private Map readFields(ObjectDescriptor descriptor, String tag, boolean payloadOnly) + throws XMLStreamException { + var fields = payloadOnly ? indexFields(layout.fields(descriptor, activeTypes, true)) + : fieldIndexes.computeIfAbsent(descriptor.symbol(), unused -> indexFields(layout.fields(descriptor, activeTypes, false))); + var data = new LinkedHashMap(); + int lastPosition = -1; + int token = xml.nextTag(); + while (!is(END_ELEMENT, token, tag)) { + String name = elementName(token); + var positioned = fields.get(name); + if (positioned == null) { + throw xml.error(INVALID_WIRE, "Unknown XTF field " + name + " on " + tag); + } + if (positioned.position() < 0) { + throw xml.error(UNSUPPORTED, "Ambiguous XTF field " + name + " on " + tag); + } + if (positioned.position() <= lastPosition) { + throw xml.error(INVALID_WIRE, "Repeated or out-of-order XTF field " + name); + } + lastPosition = positioned.position(); + var field = positioned.field(); + var value = field.embeddings().isEmpty() ? switch (field.slot()) { + case AttributeSlot attribute -> readValue(layout.type(attribute.effectiveType()), name); + case RoleSlot role -> readRole(role, name); + } : readEmbedded(field, name); + data.put(field.slot().symbol(), value); + token = xml.nextTag(); + } + return data; + } + + private TransferValue readValue(TypeDescriptor type, String tag) throws XMLStreamException { + return switch (type.kind()) { + case TEXT -> readText(((TextType) type).textKind() == TextKind.MTEXT); + case FORMATTED -> readText(false); + case BOOLEAN -> readBoolean(); + case ENUMERATION, ALIGNMENT -> readEnumeration(); + case NUMERIC -> { + attributes(false); + yield readNumber(); + } + case REFERENCE -> { + var result = reference(attributes(false, "REF", "BID"), ((ReferenceType) type).external(), false); + require(END_ELEMENT, xml.nextTag(), tag); + yield result; + } + case STRUCTURE_VALUE -> { + attributes(false); + var result = readStructure(((StructureValueType) type).targetRoots(), xml.nextTag()); + require(END_ELEMENT, xml.nextTag(), tag); + yield result; + } + case COLLECTION -> readCollection((CollectionType) type, tag); + case BLACKBOX -> readBlackbox((BlackboxType) type, tag); + case OID -> readOid((OidType) type, tag); + case CLASS_VALUE -> new TransferValue.ClassName(readModelPath(false)); + case ATTRIBUTE_PATH -> new TransferValue.AttributePath(readModelPath(true)); + case COORDINATE, GEOMETRY -> readGeometry(type, tag); + default -> throw xml.error(UNSUPPORTED, "Unsupported XTF type " + type.kind() + " for " + tag); + }; + } + + private TransferValue.Oid readOid(OidType type, String tag) throws XMLStreamException { + String value = identifier(required(attributes(false, "OID"), "OID")); + if (!Xtf23Lexemes.oidValue(type, value)) { + throw xml.error(INVALID_WIRE, "Invalid XTF OID value for " + type.oidKind()); + } + require(END_ELEMENT, xml.nextTag(), tag); + return new TransferValue.Oid(value); + } + + private QualifiedName readModelPath(boolean attribute) throws XMLStreamException { + attributes(false); + String value = xml.text(); + if (value.length() > Xtf23Lexemes.MAX_MODEL_PATH_LENGTH) { + throw xml.error(LIMIT_EXCEEDED, "XTF model path exceeds the length limit"); + } + var segments = List.of(value.split("\\.", -1)); + if (!Xtf23Lexemes.modelPath(segments, attribute)) { + throw xml.error(INVALID_WIRE, "Invalid qualified XTF model path"); + } + return new QualifiedName(segments); + } + + private TransferValue readBlackbox(BlackboxType type, String tag) throws XMLStreamException { + attributes(false); + int token = xml.nextTag(); + var value = switch (type.blackboxKind()) { + case BINARY -> { + require(START_ELEMENT, token, "BINBLBOX"); + attributes(false); + String encoded = xml.text(); + var compact = new StringBuilder(encoded.length()); + for (int i = 0; i < encoded.length(); i++) { + char c = encoded.charAt(i); + if (c != ' ' && c != '\r' && c != '\n' && c != '\t') { + compact.append(c); + } + } + String base64 = compact.toString(); + byte[] decoded; + try { + decoded = Base64.getDecoder().decode(base64); + } catch (IllegalArgumentException failure) { + throw xml.error(INVALID_WIRE, "Invalid XTF BINBLBOX Base64"); + } + if (!Base64.getEncoder().encodeToString(decoded).equals(base64)) { + throw xml.error(INVALID_WIRE, "Invalid padding or unused bits in XTF BINBLBOX Base64"); + } + yield new TransferValue.Binary(decoded); + } + case XML -> { + require(START_ELEMENT, token, "XMLBLBOX"); + attributes(false); + yield new TransferValue.XmlFragment(Xtf23XmlFragment.read(xml)); + } + }; + require(END_ELEMENT, xml.nextTag(), tag); + return value; + } + + private TransferValue.Text readText(boolean multiline) throws XMLStreamException { + attributes(false); + String value = xml.text(); + if (!multiline && !Xtf23Lexemes.singleLine(value)) { + throw xml.error(INVALID_WIRE, "Expected single-line XTF text"); + } + return new TransferValue.Text(value); + } + + private TransferValue.BooleanValue readBoolean() throws XMLStreamException { + attributes(false); + return switch (xml.text()) { + case "true" -> new TransferValue.BooleanValue(true); + case "false" -> new TransferValue.BooleanValue(false); + default -> throw xml.error(INVALID_WIRE, "Expected XTF BOOLEAN true or false"); + }; + } + + private TransferValue.Enumeration readEnumeration() throws XMLStreamException { + attributes(false); + String value = xml.text(); + if (!Xtf23Lexemes.enumeration(value)) { + throw xml.error(INVALID_WIRE, "Invalid XTF enumeration path"); + } + return new TransferValue.Enumeration(List.of(value.split("\\."))); + } + + private TransferValue.NumberValue readNumber() throws XMLStreamException { + String value = xml.text(); + if (value.length() > Xtf23Lexemes.MAX_NUMERIC_LENGTH) { + throw xml.error(LIMIT_EXCEEDED, "XTF numeric lexeme exceeds the numeric length limit"); + } + if (Xtf23Lexemes.nonDecimalNumber(value)) { + throw xml.error(UNSUPPORTED, "Nondecimal XTF numbers are unsupported"); + } + if (!Xtf23Lexemes.decimal(value)) { + throw xml.error(INVALID_WIRE, "Invalid XTF decimal value"); + } + try { + return Xtf23Lexemes.number(value); + } catch (NumberFormatException e) { + throw xml.error(LIMIT_EXCEEDED, "XTF decimal exceeds the supported exact exponent range"); + } + } + + private TransferGeometry readGeometry(TypeDescriptor type, String tag) throws XMLStreamException { + attributes(false); + int token = xml.nextTag(); + TransferGeometry geometry = switch (type) { + case CoordinateType coordinate -> readCoordinate(token, layout.dimensions(coordinate)); + case GeometryType line -> { + var curve = layout.curveEncoding(line); + yield switch (line.geometryKind()) { + case POLYLINE -> readPolyline(token, curve); + case SURFACE, AREA -> readSurface(token, curve); + default -> throw xml.error(UNSUPPORTED, "XTF 2.3 does not encode " + line.geometryKind()); + }; + } + default -> throw new IllegalStateException("Expected a plan geometry type"); + }; + require(END_ELEMENT, xml.nextTag(), tag); + return geometry; + } + + private TransferGeometry.Coordinate readCoordinate(int token, int dimensions) throws XMLStreamException { + require(START_ELEMENT, token, "COORD"); + attributes(false); + var coordinate = readPosition("C", dimensions); + require(END_ELEMENT, xml.nextTag(), "COORD"); + return coordinate; + } + + private TransferGeometry.Coordinate readPosition(String prefix, int dimensions) throws XMLStreamException { + var ordinates = new ArrayList(dimensions); + for (int axis = 1; axis <= dimensions; axis++) { + require(START_ELEMENT, xml.nextTag(), prefix + axis); + attributes(false); + ordinates.add(readNumber()); + } + return new TransferGeometry.Coordinate(ordinates); + } + + private TransferGeometry.Polyline readPolyline(int token, Xtf23Layout.CurveEncoding curve) throws XMLStreamException { + require(START_ELEMENT, token, "POLYLINE"); + attributes(false); + token = xml.nextTag(); + rejectGeometryExtension(token); + Optional lineAttributes = Optional.empty(); + if (!curve.lineAttributeRoots().isEmpty()) { + require(START_ELEMENT, token, "LINEATTR"); + attributes(false); + lineAttributes = Optional.of(readStructure(curve.lineAttributeRoots(), xml.nextTag())); + require(END_ELEMENT, xml.nextTag(), "LINEATTR"); + token = xml.nextTag(); + rejectGeometryExtension(token); + } + var start = readCoordinate(token, curve.dimensions()); + var segments = new ArrayList(); + token = xml.nextTag(); + while (!is(END_ELEMENT, token, "POLYLINE")) { + rejectGeometryExtension(token); + String segment = elementName(token); + segments.add(switch (segment) { + case "COORD" -> { + if (!curve.straights()) { + throw xml.error(INVALID_WIRE, "Straight segment is not declared by the line type"); + } + yield new TransferGeometry.Straight(readCoordinate(token, curve.dimensions())); + } + case "ARC" -> { + if (!curve.arcs()) { + throw xml.error(INVALID_WIRE, "ARC is not declared by the line type"); + } + yield readArc(curve.dimensions()); + } + default -> throw xml.error(INVALID_WIRE, "Unexpected XTF line segment " + segment); + }); + token = xml.nextTag(); + } + if (segments.isEmpty()) { + throw xml.error(INVALID_WIRE, "XTF POLYLINE requires at least one segment after its start position"); + } + return new TransferGeometry.Polyline(List.of(new TransferGeometry.CurvePart(start, segments)), lineAttributes); + } + + private TransferGeometry.Arc readArc(int dimensions) throws XMLStreamException { + attributes(false); + var end = readPosition("C", dimensions); + var support = readPosition("A", 2); + Optional radius = Optional.empty(); + int token = xml.nextTag(); + if (is(START_ELEMENT, token, "R")) { + attributes(false); + var value = readNumber(); + if (value.exactValue().signum() == 0) { + throw xml.error(INVALID_WIRE, "XTF ARC radius must be nonzero"); + } + radius = Optional.of(value); + token = xml.nextTag(); + } + require(END_ELEMENT, token, "ARC"); + return new TransferGeometry.Arc(support, end, radius); + } + + private TransferGeometry.Surface readSurface(int token, Xtf23Layout.CurveEncoding curve) throws XMLStreamException { + require(START_ELEMENT, token, "SURFACE"); + attributes(false); + var shell = readBoundary(xml.nextTag(), curve); + var holes = new ArrayList(); + token = xml.nextTag(); + while (!is(END_ELEMENT, token, "SURFACE")) { + holes.add(readBoundary(token, curve)); + token = xml.nextTag(); + } + return new TransferGeometry.Surface(shell, holes); + } + + private TransferGeometry.Ring readBoundary(int token, Xtf23Layout.CurveEncoding curve) throws XMLStreamException { + rejectGeometryExtension(token); + require(START_ELEMENT, token, "BOUNDARY"); + attributes(false); + var lines = new ArrayList(); + token = xml.nextTag(); + while (!is(END_ELEMENT, token, "BOUNDARY")) { + lines.add(readPolyline(token, curve)); + token = xml.nextTag(); + } + if (lines.isEmpty()) { + throw xml.error(INVALID_WIRE, "XTF BOUNDARY requires at least one POLYLINE"); + } + return new TransferGeometry.Ring(lines); + } + + private void rejectGeometryExtension(int token) { + if (is(START_ELEMENT, token, "CLIPPED")) { + throw xml.error(UNSUPPORTED, "Unsupported XTF geometry element " + elementName(token)); + } + } + + private TransferValue.Structured readStructure(List roots, int token) throws XMLStreamException { + String tag = elementName(token); + var descriptor = objects.get(tag); + if (descriptor == null || descriptor.kind() != SymbolKind.STRUCTURE + || descriptor.modifiers().contains(Modifier.ABSTRACT) || !layout.matchesRoots(descriptor, roots)) { + throw xml.error(INVALID_WIRE, "Unexpected concrete XTF structure " + tag); + } + attributes(false); + return new TransferValue.Structured(descriptor.symbol(), readFields(descriptor, tag, false)); + } + + private TransferValue.Repeated readCollection(CollectionType collection, String tag) throws XMLStreamException { + attributes(false); + if (!(layout.type(new TypeRef(plan.snapshotId(), collection.elementType())) instanceof StructureValueType structure)) { + throw xml.error(UNSUPPORTED, "XTF 2.3 collections require structure values"); + } + var values = new ArrayList(); + int token = xml.nextTag(); + while (!is(END_ELEMENT, token, tag)) { + values.add(readStructure(structure.targetRoots(), token)); + token = xml.nextTag(); + } + if (values.isEmpty()) { + throw xml.error(INVALID_WIRE, "Present XTF collection requires at least one structure"); + } + return new TransferValue.Repeated(collection.collectionKind(), values); + } + + private TransferValue.Reference readRole(RoleSlot role, String tag) throws XMLStreamException { + var value = reference(attributes(false, "REF", "BID", "ORDER_POS"), role.role().external(), role.role().ordered()); + require(END_ELEMENT, xml.nextTag(), tag); + return value; + } + + private TransferValue readEmbedded(Xtf23Layout.Field field, String tag) throws XMLStreamException { + var attributes = attributes(false, "REF", "BID", "ORDER_POS"); + var variants = field.embeddings(); + var first = variants.getFirst(); + if (variants.stream().anyMatch(value -> !value.rootOpposite().symbol().equals(first.rootOpposite().symbol()))) { + throw xml.error(UNSUPPORTED, "Ambiguous embedded XTF role " + tag); + } + int token = xml.nextTag(); + if (is(END_ELEMENT, token, tag)) { + var role = first.opposite().role(); + if (variants.stream().anyMatch(value -> value.opposite().role().external() != role.external() + || value.opposite().role().ordered() != role.ordered())) { + throw xml.error(UNSUPPORTED, "Embedded XTF role needs a concrete association tag: " + tag); + } + return reference(attributes, role.external(), role.ordered()); + } + String associationTag = elementName(token); + var variant = variants.stream().filter(value -> value.association().name().declared().toString().equals(associationTag)) + .findFirst().orElseThrow(() -> xml.error(INVALID_WIRE, "Unexpected embedded XTF association " + associationTag)); + attributes(false); + var role = variant.opposite(); + var reference = reference(attributes, role.role().external(), role.role().ordered()); + var fields = readFields(variant.association(), associationTag, true); + require(END_ELEMENT, xml.nextTag(), tag); + var values = new LinkedHashMap(); + values.put(role.symbol(), reference); + values.putAll(fields); + return new TransferValue.Structured(variant.association().symbol(), values); + } + + private TransferValue.Reference reference(Map attributes, boolean external, boolean ordered) { + String ref = identifier(required(attributes, "REF")); + ObjectKey key = stableId(ref).map(ObjectOid::new).orElseGet(() -> new TransferId(ref, IdOrigin.SOURCE)); + var targetBasket = Optional.ofNullable(attributes.get("BID")).map(this::basketId); + var position = Optional.ofNullable(attributes.get("ORDER_POS")).map(value -> { + if (value.length() > Xtf23Lexemes.MAX_NUMERIC_LENGTH) { + throw xml.error(LIMIT_EXCEEDED, "XTF ORDER_POS exceeds the numeric length limit"); + } + if (!ordered || !Xtf23Lexemes.POSITION.matcher(value).matches()) { + throw xml.error(INVALID_WIRE, "ORDER_POS requires an ordered role and a positive integer"); + } + var parsed = new BigInteger(value); + if (parsed.signum() <= 0) { + throw xml.error(INVALID_WIRE, "ORDER_POS requires a positive integer"); + } + return parsed; + }); + return new TransferValue.Reference(key, targetBasket, external, position); + } + + private TopicDescriptor topic(String name) { + var topic = topics.get(name); + if (topic == null) { + throw xml.error(INVALID_WIRE, "Unknown XTF topic " + name); + } + return topic; + } + + private BasketId basketId(String value) { + return new BasketId(identifier(value), IdOrigin.SOURCE); + } + + private String identifier(String value) { + if (value.length() > MAX_IDENTIFIER) { + throw xml.error(LIMIT_EXCEEDED, "XTF identifier exceeds " + MAX_IDENTIFIER + " characters"); + } + if (!XML_ID.matcher(value).matches()) { + throw xml.error(INVALID_WIRE, "Invalid XTF identifier"); + } + int separator = value.indexOf(':'); + if (separator >= 0 && !oidSpaces.containsKey(value.substring(0, separator))) { + throw xml.error(INVALID_WIRE, "Unknown XTF OIDSPACE prefix"); + } + return value; + } + + static Optional stableId(String value) { + return value.indexOf(':') >= 0 ? Optional.of(value) : Optional.empty(); + } + + private Consistency consistency(Map attributes) { + try { + return Consistency.valueOf(attributes.getOrDefault("CONSISTENCY", "COMPLETE")); + } catch (IllegalArgumentException e) { + throw xml.error(INVALID_WIRE, "Unknown XTF CONSISTENCY value"); + } + } + + private boolean is(int event, int token, String tag) { + var cursor = xml.cursor(); + return event == token && NAMESPACE.equals(cursor.getNamespaceURI()) && tag.equals(cursor.getLocalName()); + } + + private void require(int event, int token, String tag) { + if (!is(event, token, tag)) { + throw xml.error(INVALID_WIRE, "Expected " + (event == END_ELEMENT ? "end of " : "") + tag); + } + } + + private String elementName(int token) { + var cursor = xml.cursor(); + if (token != START_ELEMENT || !NAMESPACE.equals(cursor.getNamespaceURI())) { + throw xml.error(INVALID_WIRE, "Expected an element in the XTF 2.3 namespace"); + } + return cursor.getLocalName(); + } + + private Map attributes(boolean allowSchemaLocation, String... names) { + var allowed = List.of(names); + var values = new HashMap(); + var cursor = xml.cursor(); + for (int index = 0; index < cursor.getAttributeCount(); index++) { + var attribute = cursor.getAttributeName(index); + String name = attribute.getLocalPart(); + String value = cursor.getAttributeValue(index); + if (value.length() > Xtf23Input.MAX_TEXT) { + throw xml.error(LIMIT_EXCEEDED, "XTF attribute exceeds the text limit"); + } + if (allowSchemaLocation && XMLConstants.W3C_XML_SCHEMA_INSTANCE_NS_URI.equals(attribute.getNamespaceURI()) + && name.equals("schemaLocation")) { + continue; + } + if (!attribute.getNamespaceURI().isEmpty() || !allowed.contains(name)) { + throw xml.error(INVALID_WIRE, "Unexpected XTF attribute " + attribute); + } + if (!Xtf23Lexemes.singleLine(value)) { + throw xml.error(INVALID_WIRE, "XTF attribute " + name + " must be single-line text"); + } + values.put(name, value); + } + return values; + } + + private String required(Map attributes, String name) { + var value = attributes.get(name); + if (value == null) { + throw xml.error(INVALID_WIRE, "Missing XTF attribute " + name); + } + return value; + } + + private enum State { + BEFORE_TRANSFER, BETWEEN_BASKETS, IN_BASKET, DONE + } +} diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Writer.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Writer.java new file mode 100644 index 00000000..b06b3ec2 --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Writer.java @@ -0,0 +1,600 @@ +package guru.interlis.xtf23; + +import guru.interlis.execution.CancellationToken; +import guru.interlis.model.BlackboxType; +import guru.interlis.model.CollectionType; +import guru.interlis.model.CoordinateType; +import guru.interlis.model.GeometryType; +import guru.interlis.model.LanguageVersion; +import guru.interlis.model.ModelSymbol; +import guru.interlis.model.Modifier; +import guru.interlis.model.OidType; +import guru.interlis.model.QualifiedName; +import guru.interlis.model.ReferenceType; +import guru.interlis.model.StructureValueType; +import guru.interlis.model.SymbolId; +import guru.interlis.model.SymbolKind; +import guru.interlis.model.SymbolRef; +import guru.interlis.model.TextKind; +import guru.interlis.model.TextType; +import guru.interlis.model.TypeDescriptor; +import guru.interlis.model.TypeRef; +import guru.interlis.transfer.AttributeSlot; +import guru.interlis.transfer.ObjectDescriptor; +import guru.interlis.transfer.RoleSlot; +import guru.interlis.transfer.TransferEvent; +import guru.interlis.transfer.TransferPlan; +import guru.interlis.transfer.TransferValue; +import guru.interlis.transfer.TransferWriter; + +import javax.xml.stream.XMLStreamException; +import java.io.OutputStream; +import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static guru.interlis.transfer.TransferEvent.*; +import static guru.interlis.transfer.TransferGeometry.*; +import static guru.interlis.transfer.TransferIdentity.*; +import static guru.interlis.transfer.TransferValue.*; +import static guru.interlis.xtf23.Xtf23Exception.Reason.*; +import static guru.interlis.xtf23.Xtf23Output.error; + +/** + * Writes one XTF 2.3 FULL transfer in model order to a caller-owned stream. Events + * are written synchronously. Successful {@link EndTransfer} completes and flushes + * the XML document. Close releases the writer without closing the stream or + * completing an abandoned document. A destination failure after output begins + * can leave partial XML and makes the writer terminal. + * + *

Host source identities and the source/synthetic provenance of IDs are not + * encoded in XTF. Transport IDs, stable OID spellings and references must agree + * with the supplied OIDSPACE declarations.

+ */ +public final class Xtf23Writer extends TransferWriter { + private final TransferPlan plan; + private final Xtf23Layout layout; + private final Xtf23Output xml; + private final List oidSpaces; + private final Set oidAliases; + private final Xtf23IdentityScope identities = new Xtf23IdentityScope(); + private final Map> fields = new HashMap<>(); + private Set activeTypes = Set.of(); + + /** Creates a writer with a nonnull plan and output and no OIDSPACE declarations. */ + public Xtf23Writer(TransferPlan plan, OutputStream output) { + this(plan, output, List.of()); + } + + /** + * Creates a writer with a nonnull plan and caller-owned output. + * The nonnull list is copied in declaration order and must have unique aliases. + * The constructor does not write to the stream. + */ + public Xtf23Writer(TransferPlan plan, OutputStream output, List oidSpaces) { + super(plan); + this.plan = plan; + this.layout = new Xtf23Layout(plan); + this.xml = new Xtf23Output(output); + this.oidSpaces = List.copyOf(oidSpaces); + var aliases = new HashSet(); + if (this.oidSpaces.size() > Xtf23Reader.MAX_OID_SPACES) { + throw new IllegalArgumentException("Too many XTF OIDSPACE declarations"); + } + for (var space : this.oidSpaces) { + if (!aliases.add(space.alias())) { + throw new IllegalArgumentException("Repeated XTF OIDSPACE alias " + space.alias()); + } + } + oidAliases = Set.copyOf(aliases); + } + + @Override + protected void writeEvent(TransferEvent event, CancellationToken cancellation) { + xml.begin(cancellation); + try { + switch (event) { + case StartTransfer start -> startTransfer(start); + case StartBasket start -> startBasket(start); + case ObjectEvent occurrence -> writeObject(occurrence); + case EndBasket ignored -> { + xml.end(); + clearBasket(); + } + case EndTransfer ignored -> { + xml.end(); // DATASECTION + xml.end(); // TRANSFER + xml.endDocument(); + identities.release(); + } + } + cancellation.throwIfCancellationRequested(); + } catch (XMLStreamException failure) { + clearTransfer(); + throw Xtf23Output.translate(failure); + } catch (RuntimeException failure) { + clearTransfer(); + throw failure; + } finally { + xml.finish(); + } + } + + @Override + protected void release() { + clearTransfer(); + xml.close(); + } + + private void clearTransfer() { + clearBasket(); + identities.release(); + } + + private void clearBasket() { + activeTypes = Set.of(); + fields.clear(); + } + + private void startTransfer(StartTransfer start) throws XMLStreamException { + if (start.models().isEmpty()) { + throw error(INVALID_WIRE, "XTF MODELS requires at least one MODEL"); + } + var sender = required(start.sender(), "SENDER"); + xml.open(); + xml.start("HEADERSECTION"); + xml.attribute("VERSION", "2.3"); + xml.attribute("SENDER", sender); + xml.start("MODELS"); + var seen = new HashSet(); + for (var declaration : start.models()) { + if (!seen.add(declaration.model())) { + throw error(INVALID_WIRE, "Repeated XTF model declaration"); + } + var model = plan.model(declaration.model()).orElseThrow(); + var symbol = (ModelSymbol) plan.symbol(declaration.model()).known().orElseThrow(); + if (model.languageVersion() != LanguageVersion.ILI23) { + throw error(UNSUPPORTED, "Model " + model.name().declared() + " does not use INTERLIS 2.3"); + } + if (symbol.translation().isPresent()) { + throw error(UNSUPPORTED, "Translated models require an XTF ALIAS table"); + } + String version = required(declaration.modelVersion(), "MODEL VERSION"); + String issuer = required(declaration.issuer(), "MODEL URI"); + if (!issuer.equals(symbol.issuer())) { + throw error(INVALID_WIRE, "XTF declaration conflicts with model " + model.name().declared()); + } + xml.start("MODEL"); + xml.attribute("NAME", model.name().declared().toString()); + xml.attribute("VERSION", version); + xml.attribute("URI", issuer); + xml.end(); + } + xml.end(); // MODELS + if (!oidSpaces.isEmpty()) { + xml.start("OIDSPACES"); + for (var space : oidSpaces) { + xml.start("OIDSPACE"); + xml.attribute("NAME", space.alias()); + xml.attribute("OIDDOMAIN", space.domain()); + xml.end(); + } + xml.end(); + } + if (start.comment().isPresent()) { + textElement("COMMENT", start.comment().orElseThrow()); + } + xml.end(); // HEADERSECTION + xml.start("DATASECTION"); + } + + private void startBasket(StartBasket start) throws XMLStreamException { + if (start.kind() != BasketKind.FULL || !start.domainBindings().isEmpty()) { + throw error(UNSUPPORTED, "Only XTF 2.3 FULL baskets without domain bindings are supported"); + } + if (start.startState().isPresent() || start.endState().isPresent()) { + throw error(INVALID_WIRE, "A FULL basket cannot carry incremental state"); + } + String bid = identifier(start.id().value()); + if (!Xtf23Reader.stableId(bid).equals(start.stableOid())) { + throw error(INVALID_WIRE, "Stable basket identity must agree with the prefixed BID"); + } + if (!identities.claim(start.id())) { + throw error(DUPLICATE_ID, "XTF FULL BID/TID collision for " + bid); + } + var topic = plan.topic(start.topic()).orElseThrow(); + var types = new HashSet<>(topic.objectTypes()); + var additional = start.additionalTopics(); + if (new HashSet<>(additional).size() != additional.size()) { + throw error(INVALID_WIRE, "Repeated XTF extension topic"); + } + var names = additional.stream().map(ref -> { + var extension = plan.topic(ref).orElseThrow(); + types.addAll(extension.objectTypes()); + return extension.name().declared().toString(); + }).toList(); + activeTypes = Set.copyOf(types); + xml.start(topic.name().declared().toString()); + xml.attribute("BID", bid); + if (!names.isEmpty()) { + xml.attribute("TOPICS", String.join(",", names)); + } + writeConsistency(start.consistency()); + } + + private void writeObject(ObjectEvent occurrence) throws XMLStreamException { + var object = occurrence.object(); + var descriptor = plan.object(object.type()).orElseThrow(); + if (layout.embedded(descriptor).isPresent()) { + throw error(INVALID_WIRE, "Embedded XTF association cannot occur as a standalone object"); + } + if (occurrence.operation().isPresent()) { + throw error(INVALID_WIRE, "A FULL object cannot carry an incremental OPERATION"); + } + var identity = object.identity(); + var tid = identity.tid(); + boolean requiresTid = descriptor.kind() == SymbolKind.CLASS || layout.identified(descriptor); + if (requiresTid != tid.isPresent()) { + throw error(INVALID_WIRE, requiresTid ? "Missing XTF TID" : "A link without its own identification cannot carry TID"); + } + var wireTid = tid.map(id -> identifier(id.value())); + if (!wireTid.flatMap(Xtf23Reader::stableId).equals(identity.oid().map(ObjectOid::value))) { + throw error(INVALID_WIRE, "Stable object identity must agree with the prefixed TID"); + } + if (tid.isPresent() && !identities.claim(tid.orElseThrow())) { + throw error(DUPLICATE_ID, "XTF FULL BID/TID collision for " + wireTid.orElseThrow()); + } + xml.start(descriptor.name().declared().toString()); + if (wireTid.isPresent()) { + xml.attribute("TID", wireTid.orElseThrow()); + } + if (identity.originalBasket().isPresent()) { + xml.attribute("BID", identifier(identity.originalBasket().orElseThrow().value())); + } + writeConsistency(occurrence.consistency()); + writeFields(descriptor, object.data(), Optional.empty()); + xml.end(); + } + + private void writeConsistency(Consistency consistency) throws XMLStreamException { + if (consistency != Consistency.COMPLETE) { + xml.attribute("CONSISTENCY", consistency.name()); + } + } + + private void writeFields(ObjectDescriptor descriptor, Structured data, Optional embeddedReference) + throws XMLStreamException { + var available = embeddedReference.isPresent() ? layout.fields(descriptor, activeTypes, true) + : fields.computeIfAbsent(descriptor.symbol(), key -> layout.fields(descriptor, activeTypes, false)); + var remaining = new HashSet<>(data.values().keySet()); + embeddedReference.ifPresent(remaining::remove); + var names = new HashMap(); + for (var field : available) { + names.merge(field.name(), 1, Integer::sum); + } + for (var field : available) { + var value = data.values().get(field.slot().symbol()); + if (value == null) { + continue; + } + if (!remaining.remove(field.slot().symbol()) || names.get(field.name()) != 1) { + throw error(UNSUPPORTED, "Ambiguous XTF field " + field.name()); + } + if (value == Undefined.INSTANCE) { + throw error(UNSUPPORTED, "XTF has no explicit undefined value for " + field.name()); + } + xml.start(field.name()); + if (!field.embeddings().isEmpty()) { + writeEmbedded(field, value); + } else { + switch (field.slot()) { + case AttributeSlot attribute -> writeValue(layout.type(attribute.effectiveType()), value); + case RoleSlot role -> writeReference(requireValue(value, Reference.class), + role.role().external(), role.role().ordered()); + } + } + xml.end(); + } + if (!remaining.isEmpty()) { + throw error(INVALID_WIRE, "Object supplies a field outside its XTF layout"); + } + } + + private void writeValue(TypeDescriptor type, TransferValue value) throws XMLStreamException { + switch (type.kind()) { + case TEXT -> writeText(requireValue(value, Text.class), ((TextType) type).textKind() == TextKind.MTEXT); + case FORMATTED -> writeText(requireValue(value, Text.class), false); + case BOOLEAN -> xml.text(Boolean.toString(requireValue(value, BooleanValue.class).value())); + case ENUMERATION, ALIGNMENT -> writeEnumeration(requireValue(value, Enumeration.class)); + case NUMERIC -> xml.text(number(requireValue(value, NumberValue.class))); + case REFERENCE -> writeReference(requireValue(value, Reference.class), ((ReferenceType) type).external(), false); + case STRUCTURE_VALUE -> writeStructure(((StructureValueType) type).targetRoots(), requireValue(value, Structured.class)); + case COLLECTION -> writeCollection((CollectionType) type, requireValue(value, Repeated.class)); + case BLACKBOX -> writeBlackbox((BlackboxType) type, value); + case OID -> writeOid((OidType) type, requireValue(value, Oid.class)); + case CLASS_VALUE -> writeModelPath(requireValue(value, ClassName.class).name(), false); + case ATTRIBUTE_PATH -> writeModelPath(requireValue(value, AttributePath.class).name(), true); + case COORDINATE, GEOMETRY -> writeGeometry(type, value); + default -> throw error(UNSUPPORTED, "Unsupported XTF type " + type.kind()); + } + } + + private void writeOid(OidType type, Oid value) throws XMLStreamException { + String lexeme = identifier(value.lexeme()); + if (!Xtf23Lexemes.oidValue(type, lexeme)) { + throw error(INVALID_WIRE, "Invalid XTF OID value for " + type.oidKind()); + } + xml.attribute("OID", lexeme); + } + + private void writeModelPath(QualifiedName name, boolean attribute) throws XMLStreamException { + if (!Xtf23Lexemes.modelPath(name.segments(), attribute)) { + throw error(INVALID_WIRE, "Invalid qualified XTF model path"); + } + String value = name.toString(); + if (value.length() > Xtf23Lexemes.MAX_MODEL_PATH_LENGTH) { + throw error(LIMIT_EXCEEDED, "XTF model path exceeds the length limit"); + } + xml.text(value); + } + + private void writeBlackbox(BlackboxType type, TransferValue value) throws XMLStreamException { + switch (type.blackboxKind()) { + case BINARY -> { + var binary = requireValue(value, Binary.class); + if (binary.size() > Xtf23Input.MAX_TEXT / 4 * 3) { + throw error(LIMIT_EXCEEDED, "XTF BINBLBOX exceeds the Base64 text limit"); + } + textElement("BINBLBOX", Base64.getEncoder().encodeToString(binary.bytes())); + } + case XML -> { + xml.start("XMLBLBOX"); + xml.fragment(requireValue(value, XmlFragment.class).xml()); + xml.end(); + } + } + } + + private void writeText(Text text, boolean multiline) throws XMLStreamException { + if (!multiline && !Xtf23Lexemes.singleLine(text.value())) { + throw error(INVALID_WIRE, "Expected single-line XTF text"); + } + xml.text(text.value()); + } + + private void writeEnumeration(Enumeration enumeration) throws XMLStreamException { + String path = String.join(".", enumeration.path()); + if (!Xtf23Lexemes.enumeration(path) || !List.of(path.split("\\.")).equals(enumeration.path())) { + throw error(INVALID_WIRE, "Invalid XTF enumeration path"); + } + xml.text(path); + } + + private String number(NumberValue number) { + String lexeme = number.lexeme(); + if (lexeme.length() > Xtf23Lexemes.MAX_NUMERIC_LENGTH) { + throw error(LIMIT_EXCEEDED, "XTF numeric lexeme exceeds the numeric length limit"); + } + if (!Xtf23Lexemes.decimal(lexeme)) { + throw error(INVALID_WIRE, "Invalid XTF decimal lexeme"); + } + try { + if (!Xtf23Lexemes.number(lexeme).equals(number)) { + throw error(INVALID_WIRE, "XTF numeric lexeme differs from its exact value"); + } + } catch (NumberFormatException e) { + throw error(LIMIT_EXCEEDED, "XTF decimal exceeds the supported exact exponent range"); + } + return lexeme; + } + + private void writeStructure(List roots, Structured value) throws XMLStreamException { + var descriptor = plan.object(value.type()).orElseThrow(() -> error(INVALID_WIRE, "Unknown XTF structure type")); + if (descriptor.kind() != SymbolKind.STRUCTURE || descriptor.modifiers().contains(Modifier.ABSTRACT) + || !layout.matchesRoots(descriptor, roots)) { + throw error(INVALID_WIRE, "Unexpected concrete XTF structure " + descriptor.name().declared()); + } + xml.start(descriptor.name().declared().toString()); + writeFields(descriptor, value, Optional.empty()); + xml.end(); + } + + private void writeCollection(CollectionType type, Repeated collection) throws XMLStreamException { + if (collection.kind() != type.collectionKind()) { + throw error(INVALID_WIRE, "XTF collection kind differs from the model"); + } + if (collection.values().isEmpty()) { + throw error(UNSUPPORTED, "XTF cannot encode an explicit empty BAG or LIST"); + } + if (!(layout.type(new TypeRef(plan.snapshotId(), type.elementType())) instanceof StructureValueType structure)) { + throw error(UNSUPPORTED, "XTF 2.3 collections require structure values"); + } + for (var value : collection.values()) { + writeStructure(structure.targetRoots(), requireValue(value, Structured.class)); + } + } + + private void writeEmbedded(Xtf23Layout.Field field, TransferValue value) throws XMLStreamException { + var variants = field.embeddings(); + var first = variants.getFirst(); + if (variants.stream().anyMatch(variant -> !variant.rootOpposite().symbol().equals(first.rootOpposite().symbol()))) { + throw error(UNSUPPORTED, "Ambiguous embedded XTF role " + field.name()); + } + if (value instanceof Reference reference) { + var role = first.opposite().role(); + if (variants.stream().anyMatch(variant -> variant.opposite().role().external() != role.external() + || variant.opposite().role().ordered() != role.ordered())) { + throw error(UNSUPPORTED, "Embedded XTF role needs a concrete association type: " + field.name()); + } + writeReference(reference, role.external(), role.ordered()); + return; + } + var payload = requireValue(value, Structured.class); + var variant = variants.stream().filter(candidate -> candidate.association().symbol().equals(payload.type())) + .findFirst().orElseThrow(() -> error(INVALID_WIRE, "Unexpected embedded XTF association type")); + var role = variant.opposite(); + var reference = requireValue(payload.values().get(role.symbol()), Reference.class); + writeReference(reference, role.role().external(), role.role().ordered()); + xml.start(variant.association().name().declared().toString()); + writeFields(variant.association(), payload, Optional.of(role.symbol())); + xml.end(); + } + + private void writeReference(Reference reference, boolean external, boolean ordered) throws XMLStreamException { + var target = reference.target(); + String ref = identifier(target.value()); + if ((target instanceof ObjectOid) != Xtf23Reader.stableId(ref).isPresent() + || reference.external() != external) { + throw error(INVALID_WIRE, "XTF reference identity or EXTERNAL flag conflicts with its encoding"); + } + xml.attribute("REF", ref); + if (reference.targetBasket().isPresent()) { + xml.attribute("BID", identifier(reference.targetBasket().orElseThrow().value())); + } + if (reference.orderPosition().isPresent()) { + if (!ordered) { + throw error(INVALID_WIRE, "ORDER_POS requires an ordered role"); + } + var position = reference.orderPosition().orElseThrow(); + if (position.bitLength() > 4 * Xtf23Lexemes.MAX_NUMERIC_LENGTH) { + throw error(LIMIT_EXCEEDED, "XTF ORDER_POS exceeds the numeric length limit"); + } + String value = position.toString(); + if (value.length() > Xtf23Lexemes.MAX_NUMERIC_LENGTH) { + throw error(LIMIT_EXCEEDED, "XTF ORDER_POS exceeds the numeric length limit"); + } + xml.attribute("ORDER_POS", value); + } + } + + private void writeGeometry(TypeDescriptor type, TransferValue value) throws XMLStreamException { + switch (type) { + case CoordinateType coordinate -> writeCoordinate(requireValue(value, Coordinate.class), layout.dimensions(coordinate)); + case GeometryType line -> { + var curve = layout.curveEncoding(line); + switch (line.geometryKind()) { + case POLYLINE -> writePolyline(requireValue(value, Polyline.class), curve); + case SURFACE, AREA -> writeSurface(requireValue(value, Surface.class), curve); + default -> throw error(UNSUPPORTED, "XTF 2.3 does not encode " + line.geometryKind()); + } + } + default -> throw new IllegalStateException("Expected a plan geometry type"); + } + } + + private void writeCoordinate(Coordinate coordinate, int dimensions) throws XMLStreamException { + xml.start("COORD"); + writePosition(coordinate, "C", dimensions); + xml.end(); + } + + private void writePosition(Coordinate coordinate, String prefix, int dimensions) throws XMLStreamException { + if (coordinate.ordinates().size() != dimensions) { + throw error(INVALID_WIRE, "Coordinate dimension differs from the XTF line or coordinate type"); + } + for (int i = 0; i < dimensions; i++) { + textElement(prefix + (i + 1), number(coordinate.ordinates().get(i))); + } + } + + private void writePolyline(Polyline line, Xtf23Layout.CurveEncoding curve) throws XMLStreamException { + if (curve.lineAttributeRoots().isEmpty() && line.lineAttributes().isPresent()) { + throw error(INVALID_WIRE, "XTF LINEATTR is not declared for this line"); + } + if (!curve.lineAttributeRoots().isEmpty() && line.lineAttributes().isEmpty()) { + throw error(INVALID_WIRE, "XTF LINEATTR is required for this line"); + } + if (line.parts().size() != 1) { + throw error(UNSUPPORTED, "XTF POLYLINE requires one unclipped sequence"); + } + var part = line.parts().getFirst(); + if (part.segments().isEmpty()) { + throw error(INVALID_WIRE, "XTF POLYLINE requires at least one segment after its start position"); + } + xml.start("POLYLINE"); + if (line.lineAttributes().isPresent()) { + xml.start("LINEATTR"); + writeStructure(curve.lineAttributeRoots(), line.lineAttributes().orElseThrow()); + xml.end(); + } + writeCoordinate(part.start(), curve.dimensions()); + for (var segment : part.segments()) { + switch (segment) { + case Straight straight -> { + if (!curve.straights()) { + throw error(INVALID_WIRE, "Straight segment is not declared by the line type"); + } + writeCoordinate(straight.end(), curve.dimensions()); + } + case Arc arc -> { + if (!curve.arcs()) { + throw error(INVALID_WIRE, "ARC is not declared by the line type"); + } + xml.start("ARC"); + writePosition(arc.end(), "C", curve.dimensions()); + writePosition(arc.support(), "A", 2); + if (arc.radius().isPresent()) { + textElement("R", number(arc.radius().orElseThrow())); + } + xml.end(); + } + } + } + xml.end(); + } + + private void writeSurface(Surface surface, Xtf23Layout.CurveEncoding curve) throws XMLStreamException { + xml.start("SURFACE"); + writeBoundary(surface.shell(), curve); + for (var hole : surface.holes()) { + writeBoundary(hole, curve); + } + xml.end(); + } + + private void writeBoundary(Ring boundary, Xtf23Layout.CurveEncoding curve) throws XMLStreamException { + xml.start("BOUNDARY"); + for (var line : boundary.lines()) { + writePolyline(line, curve); + } + xml.end(); + } + + private void textElement(String name, String value) throws XMLStreamException { + xml.start(name); + xml.text(value); + xml.end(); + } + + private String identifier(String value) { + if (value.length() > Xtf23Reader.MAX_IDENTIFIER) { + throw error(LIMIT_EXCEEDED, "XTF identifier exceeds the length limit"); + } + if (!Xtf23Reader.XML_ID.matcher(value).matches()) { + throw error(INVALID_WIRE, "Invalid XTF identifier"); + } + int separator = value.indexOf(':'); + if (separator >= 0 && !oidAliases.contains(value.substring(0, separator))) { + throw error(INVALID_WIRE, "Unknown XTF OIDSPACE prefix"); + } + return value; + } + + private static String required(Optional value, String name) { + return value.orElseThrow(() -> error(INVALID_WIRE, "Missing XTF " + name)); + } + + private static T requireValue(TransferValue value, Class expected) { + if (value == Undefined.INSTANCE) { + throw error(UNSUPPORTED, "XTF has no explicit undefined value"); + } + if (!expected.isInstance(value)) { + throw error(INVALID_WIRE, "XTF field requires " + expected.getSimpleName()); + } + return expected.cast(value); + } +} diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23XmlFragment.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23XmlFragment.java new file mode 100644 index 00000000..c1ce397e --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23XmlFragment.java @@ -0,0 +1,157 @@ +package guru.interlis.xtf23; + +import guru.interlis.execution.CancellationToken; + +import javax.xml.namespace.QName; +import javax.xml.stream.XMLStreamException; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; + +import static guru.interlis.xtf23.Xtf23Exception.Reason.*; +import static javax.xml.stream.XMLStreamConstants.*; + +/** Captures payload events as self-contained XML, including namespace bindings inherited from XTF. */ +final class Xtf23XmlFragment { + private static final String WRAPPER = "blackboxFragment"; + + private Xtf23XmlFragment() { } + + static String read(Xtf23Input input) throws XMLStreamException { + return read(input, 0); + } + + static String validate(String value, int outerDepth, CancellationToken cancellation) { + String wrapped = "<" + WRAPPER + ">" + + value + ""; + var input = new Xtf23Input(new ByteArrayInputStream(wrapped.getBytes(StandardCharsets.UTF_8))); + input.begin(cancellation); + try { + input.open(); + input.nextTag(); + String checked = read(input, outerDepth); + if (input.next() != END_DOCUMENT) { + throw input.error(INVALID_WIRE, "Unexpected content after the XML fragment"); + } + return checked; + } catch (XMLStreamException failure) { + throw input.translate(failure); + } finally { + input.close(); + } + } + + private static String read(Xtf23Input input, int outerDepth) throws XMLStreamException { + var fragment = new Fragment(input); + var cursor = input.cursor(); + int depth = 0; + while (true) { + switch (input.next()) { + case START_ELEMENT -> { + boolean root = depth == 0; + if (++depth + outerDepth > Xtf23Input.MAX_DEPTH) { + throw input.error(LIMIT_EXCEEDED, "XML fragment exceeds the output depth limit"); + } + fragment.append("<"); + fragment.name(cursor.getName()); + var namespaces = input.fragmentNamespaces(root); + if (namespaces.size() + cursor.getAttributeCount() > Xtf23Input.MAX_ATTRIBUTES) { + throw input.error(LIMIT_EXCEEDED, "Too many attributes and namespace declarations in XML fragment"); + } + for (var binding : namespaces.entrySet()) { + String prefix = binding.getKey(); + fragment.append(prefix.isEmpty() ? " xmlns" : " xmlns:" + prefix); + fragment.attributeValue(binding.getValue()); + } + for (int i = 0; i < cursor.getAttributeCount(); i++) { + fragment.append(" "); + fragment.name(cursor.getAttributeName(i)); + fragment.attributeValue(cursor.getAttributeValue(i)); + } + fragment.append(">"); + } + case END_ELEMENT -> { + if (depth == 0) { + return fragment.value.toString(); + } + fragment.append(""); + depth--; + } + case CHARACTERS, CDATA, SPACE -> fragment.escape(cursor.getText(), false); + case COMMENT -> { + fragment.append(""); + } + case PROCESSING_INSTRUCTION -> { + fragment.append(""); + } + default -> throw input.error(INVALID_WIRE, "Unexpected event inside XML fragment"); + } + } + } + + private static final class Fragment { + private final Xtf23Input input; + private final StringBuilder value = new StringBuilder(); + + Fragment(Xtf23Input input) { + this.input = input; + } + + void name(QName name) { + if (!name.getPrefix().isEmpty()) { + append(name.getPrefix()); + append(":"); + } + append(name.getLocalPart()); + } + + void attributeValue(String text) { + append("=\""); + escape(text, true); + append("\""); + } + + void escape(String text, boolean attribute) { + int start = 0; + for (int i = 0; i < text.length(); i++) { + String replacement = switch (text.charAt(i)) { + case '&' -> "&"; + case '<' -> "<"; + case '>' -> ">"; + case '\r' -> " "; + case '\n' -> attribute ? " " : null; + case '\t' -> attribute ? " " : null; + case '"' -> attribute ? """ : null; + default -> null; + }; + if (replacement != null) { + append(text, start, i); + append(replacement); + start = i + 1; + } + } + append(text, start, text.length()); + } + + void append(String text) { + append(text, 0, text.length()); + } + + void append(String text, int start, int end) { + if (end - start > Xtf23Input.MAX_TEXT - value.length()) { + throw input.error(LIMIT_EXCEEDED, "XML fragment exceeds the XTF text limit"); + } + value.append(text, start, end); + } + } +} diff --git a/interlis-xtf23/src/main/java/module-info.java b/interlis-xtf23/src/main/java/module-info.java new file mode 100644 index 00000000..00baa447 --- /dev/null +++ b/interlis-xtf23/src/main/java/module-info.java @@ -0,0 +1,8 @@ +/** Streaming XML adapter for INTERLIS 2.3 transfers. */ +module guru.interlis.xtf23 { + requires java.xml; + requires guru.interlis.model; + requires transitive guru.interlis.transfer; + + exports guru.interlis.xtf23; +} diff --git a/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23BlackboxTest.java b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23BlackboxTest.java new file mode 100644 index 00000000..c5c8588e --- /dev/null +++ b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23BlackboxTest.java @@ -0,0 +1,163 @@ +package guru.interlis.xtf23; + +import guru.interlis.transfer.TransferEvent; +import guru.interlis.transfer.TransferObject; +import guru.interlis.transfer.TransferPlan; +import guru.interlis.transfer.TransferValue; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static guru.interlis.transfer.TransferEvent.ObjectEvent; +import static guru.interlis.xtf23.Xtf23Exception.Reason.*; +import static guru.interlis.xtf23.Xtf23TestData.*; +import static org.junit.jupiter.api.Assertions.*; + +class Xtf23BlackboxTest { + private static final String MODELS = """ + + """; + private static TransferPlan plan; + + @BeforeAll + static void compileModel() { + plan = compileInline("Boxes", """ + INTERLIS 2.3; + MODEL Boxes (en) AT "https://example.invalid/boxes" VERSION "1" = + TOPIC Data = + CLASS Item = + document : BLACKBOX XML; + payload : BLACKBOX BINARY; + END Item; + END Data; + END Boxes. + """); + } + + @Test + void keepsMixedContentAndInScopeNamespacesWithExactAttributeValues() { + String content = """ + leadingone two lasttail"""; + var events = read(transfer("" + content + "" + + " A Q I D \n /w==\t")); + var xml = assertInstanceOf(TransferValue.XmlFragment.class, field(events, "document")).xml(); + assertTrue(xml.startsWith("leadingtwo ")); + assertTrue(xml.contains("")); + assertEquals(new TransferValue.XmlFragment(""), field(events, "document")); + assertEquals(new TransferValue.Binary(new byte[0]), field(events, "payload")); + assertEquals(events, read(write(events))); + + String encoded = write(withDocument(events, new TransferValue.XmlFragment(""))); + assertTrue(encoded.contains("")); + assertEquals(new TransferValue.XmlFragment(""), + field(read(encoded), "document")); + } + + @Test + void rejectsInvalidBinaryAndWrongWrappers() { + for (String bad : List.of("Zg", "Zh==", "Zg=", "Zg===", "Z-g=", "Zg== ", "Zg==")) { + var xml = transfer("" + bad + ""); + assertEquals(INVALID_WIRE, assertThrows(Xtf23Exception.class, () -> read(xml), bad).reason()); + } + for (String wrong : List.of("", "", + "", "")) { + assertEquals(INVALID_WIRE, assertThrows(Xtf23Exception.class, () -> read(transfer(wrong))).reason()); + } + } + + @Test + void rejectsXmlDocumentInjectionEntitiesAndOversizePayloads() { + var events = read(transfer("")); + for (String bad : List.of("", "&external;", + "", "", "", "")) { + var output = new ByteArrayOutputStream(); + assertThrows(Xtf23Exception.class, + () -> writeTo(withDocument(events, new TransferValue.XmlFragment(bad)), output), bad); + assertFalse(output.toString(StandardCharsets.UTF_8).contains("&xxe;") + .replace("]> read(dtd)).reason()); + + String longText = "A".repeat(Xtf23Input.MAX_TEXT + 1); + for (String content : List.of("" + longText + "", + "" + longText + "")) { + assertEquals(LIMIT_EXCEEDED, + assertThrows(Xtf23Exception.class, () -> read(transfer(content))).reason()); + } + assertEquals(LIMIT_EXCEEDED, assertThrows(Xtf23Exception.class, + () -> write(withDocument(events, new TransferValue.XmlFragment(longText)))).reason()); + assertEquals(LIMIT_EXCEEDED, assertThrows(Xtf23Exception.class, + () -> write(withField(events, "payload", + new TransferValue.Binary(new byte[Xtf23Input.MAX_TEXT / 4 * 3 + 1])))).reason()); + String tooDeep = "".repeat(Xtf23Input.MAX_DEPTH) + "".repeat(Xtf23Input.MAX_DEPTH); + assertEquals(LIMIT_EXCEEDED, assertThrows(Xtf23Exception.class, + () -> write(withDocument(events, new TransferValue.XmlFragment(tooDeep)))).reason()); + } + + private static String transfer(String fields) { + return document(MODELS, "" + + "" + fields + ""); + } + + private static List read(String source) { + try (var reader = new Xtf23Reader(plan, input(source))) { + return readAll(reader); + } + } + + private static String write(List events) { + var output = new ByteArrayOutputStream(); + writeTo(events, output); + return output.toString(StandardCharsets.UTF_8); + } + + private static void writeTo(List events, ByteArrayOutputStream output) { + try (var writer = new Xtf23Writer(plan, output)) { + for (var event : events) { + writer.write(event, TOKEN); + } + } + } + + private static TransferValue field(List events, String name) { + var object = ((ObjectEvent) events.get(2)).object(); + return object.value(slot(object, name)).orElseThrow(); + } + + private static List withDocument(List events, TransferValue value) { + return withField(events, "document", value); + } + + private static List withField(List events, String name, TransferValue value) { + var replacement = new ArrayList<>(events); + var event = (ObjectEvent) events.get(2); + var object = event.object(); + replacement.set(2, new ObjectEvent(new TransferObject(object.identity(), + new TransferValue.Structured(object.type(), Map.of(slot(object, name), value))), + event.operation(), event.consistency())); + return replacement; + } + + private static guru.interlis.model.SymbolRef slot(TransferObject object, String name) { + return plan.object(object.type()).orElseThrow().slots().stream() + .filter(slot -> slot.name().declared().last().equals(name)).findFirst().orElseThrow().symbol(); + } +} diff --git a/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23GeometryTest.java b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23GeometryTest.java new file mode 100644 index 00000000..e2e9b714 --- /dev/null +++ b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23GeometryTest.java @@ -0,0 +1,183 @@ +package guru.interlis.xtf23; + +import guru.interlis.transfer.TransferEvent.ObjectEvent; +import guru.interlis.transfer.TransferObject; +import guru.interlis.transfer.TransferPlan; +import guru.interlis.transfer.TransferValue; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.file.Files; +import java.util.List; + +import static guru.interlis.transfer.TransferGeometry.*; +import static guru.interlis.xtf23.Xtf23Exception.Reason.*; +import static guru.interlis.xtf23.Xtf23TestData.*; +import static org.junit.jupiter.api.Assertions.*; + +final class Xtf23GeometryTest { + private static TransferPlan curves; + private static TransferPlan reference; + private static final String MODELS_CURVES = """ + + """; + private static final String START_3D = "0.000.0+1.2e0"; + + @BeforeAll + static void compileModels() throws IOException { + reference = referencePlan(); + curves = compileInline("Curves", """ + INTERLIS 2.3; + MODEL Curves (en) AT "https://example.invalid/curves" VERSION "1" = + TOPIC Data = + DOMAIN + Point2 = COORD 0.00 .. 100.00, 0.00 .. 100.00; + Point3 = COORD 0.00 .. 100.00, 0.00 .. 100.00, 0.00 .. 100.00; + CLASS Shape = + point: Point3; + trace: POLYLINE WITH (STRAIGHTS, ARCS) VERTEX Point3; + surface: SURFACE WITH (STRAIGHTS, ARCS) VERTEX Point2 WITHOUT OVERLAPS > 0.01; + straight: POLYLINE WITH (STRAIGHTS) VERTEX Point2; + END Shape; + END Data; + END Curves. + """); + } + + @Test + void preservesThreeDimensionalEndpointsAndSignedExactArcRadii() { + String arc = """ + 2.01003.0025.43211.110002.000-9.12345678901234567890 + 3.04.06.02.223.33+8.5e-1 + """; + String straight = "4.05.07.0"; + try (var reader = new Xtf23Reader(curves, input(shape("" + START_3D + "" + + START_3D + arc + straight + "")))) { + var object = objects(reader).getFirst(); + var point = assertInstanceOf(Coordinate.class, field(curves, object, "point")); + assertEquals(List.of("0.00", "0.0", "+1.2e0"), lexemes(point)); + var line = assertInstanceOf(Polyline.class, field(curves, object, "trace")); + assertEquals(1, line.parts().size()); + var part = line.parts().getFirst(); + assertEquals(point, part.start()); + assertEquals(3, part.segments().size()); + var negative = assertInstanceOf(Arc.class, part.segments().get(0)); + assertEquals(List.of("2.0100", "3.002", "5.4321"), lexemes(negative.end())); + assertEquals(List.of("1.11000", "2.000"), lexemes(negative.support())); + assertEquals(new TransferValue.NumberValue("-9.12345678901234567890", new BigDecimal("-9.12345678901234567890")), + negative.radius().orElseThrow()); + var positive = assertInstanceOf(Arc.class, part.segments().get(1)); + assertEquals("+8.5e-1", positive.radius().orElseThrow().lexeme()); + assertEquals(new BigDecimal("0.85"), positive.radius().orElseThrow().exactValue()); + assertEquals(List.of("4.0", "5.0", "7.0"), lexemes(assertInstanceOf(Straight.class, + part.segments().getLast()).end())); + } + } + + @Test + void preservesShellHolesAndPartitionIntoBoundaryLines() { + String first = line(point("0", "0"), point("10", "0"), point("10", "10")); + String second = line(point("10", "10"), point("0", "10"), point("0", "0")); + String hole = line(point("2", "2"), point("3", "2"), point("3", "3"), point("2", "2")); + String field = "" + first + second + "" + hole + + ""; + try (var reader = new Xtf23Reader(curves, input(shape(field)))) { + var surface = assertInstanceOf(Surface.class, field(curves, objects(reader).getFirst(), "surface")); + assertEquals(2, surface.shell().lines().size()); + var lines = surface.shell().lines(); + assertEquals(List.of("0", "0"), lexemes(lines.get(0).parts().getFirst().start())); + assertEquals(List.of("10", "10"), lexemes(lines.get(1).parts().getFirst().start())); + assertEquals(1, surface.holes().size()); + assertEquals(List.of("2", "2"), lexemes(surface.holes().getFirst().lines().getFirst().parts().getFirst().start())); + } + } + + @Test + void decodesAreaFixturesAndPreservesTheSelfCrossingG0Boundary() throws IOException { + for (String name : List.of("area-shared-edge.xtf", "area-overlap.xtf")) { + try (var source = Files.newInputStream(fixture(name)); var reader = new Xtf23Reader(reference, source)) { + var objects = objects(reader); + assertEquals(2, objects.size()); + for (var object : objects) { + var area = assertInstanceOf(Surface.class, field(reference, object, "shape")); + assertEquals(4, area.shell().lines().getFirst().parts().getFirst().segments().size()); + } + } + } + // Same wire points as the pinned G0-B area-self-crossing-ring case. + String crossing = line(point("10.00", "10.00"), point("20.00", "20.00"), point("20.00", "10.00"), + point("10.00", "20.00"), point("10.00", "10.00")); + String xml = document(MODELS, """ + + %s + + """.formatted(crossing)); + try (var reader = new Xtf23Reader(reference, input(xml))) { + var area = assertInstanceOf(Surface.class, field(reference, objects(reader).getFirst(), "shape")); + var vertices = area.shell().lines().getFirst().parts().getFirst().segments().stream() + .map(segment -> lexemes(segment.end())).toList(); + assertEquals(List.of(List.of("20.00", "20.00"), List.of("20.00", "10.00"), + List.of("10.00", "20.00"), List.of("10.00", "10.00")), vertices); + } + } + + @Test + void rejectsMalformedGeometryAndReportsClippedAsUnsupported() { + String two = point("1", "2"); + String validArc = "3423"; + for (String value : List.of( + "" + two + "", + "" + line(START_3D) + "", + "" + line(START_3D, validArc) + "", + "" + line(START_3D, two) + "", + "", + "", + "", + "" + line(two, validArc) + "", + "" + line(two, validArc.replace("", "0.0")) + + "", + "" + line(two, validArc.replace("", "1")) + + "")) { + assertEquals(INVALID_WIRE, failed(value), value); + } + assertEquals(UNSUPPORTED, failed("" + + START_3D + START_3D + "")); + assertEquals(LIMIT_EXCEEDED, failed("" + START_3D.replace("0.00", "1e2147483649") + "")); + } + + private static Xtf23Exception.Reason failed(String fields) { + try (var reader = new Xtf23Reader(curves, input(shape(fields)))) { + return assertThrows(Xtf23Exception.class, () -> readAll(reader)).reason(); + } + } + + private static String shape(String fields) { + return document(MODELS_CURVES, "" + + fields + ""); + } + + private static String point(String x, String y) { + return "" + x + "" + y + ""; + } + + private static String line(String... entries) { + return "" + String.join("", entries) + ""; + } + + private static List objects(Xtf23Reader reader) { + return readAll(reader).stream().filter(ObjectEvent.class::isInstance).map(ObjectEvent.class::cast) + .map(ObjectEvent::object).toList(); + } + + private static TransferValue field(TransferPlan plan, TransferObject value, String name) { + var slot = plan.object(value.type()).orElseThrow().slots().stream() + .filter(candidate -> candidate.name().declared().last().equals(name)).findFirst().orElseThrow(); + return value.value(slot.symbol()).orElseThrow(); + } + + private static List lexemes(Coordinate point) { + return point.ordinates().stream().map(TransferValue.NumberValue::lexeme).toList(); + } +} diff --git a/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23HeaderTest.java b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23HeaderTest.java new file mode 100644 index 00000000..17f9896f --- /dev/null +++ b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23HeaderTest.java @@ -0,0 +1,130 @@ +package guru.interlis.xtf23; + +import guru.interlis.source.SourceId; +import guru.interlis.transfer.TransferEvent; +import guru.interlis.transfer.TransferPlan; +import guru.interlis.transfer.TransferProtocolException; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.util.List; +import java.util.Optional; + +import static guru.interlis.xtf23.Xtf23Exception.Reason.*; +import static guru.interlis.xtf23.Xtf23TestData.*; +import static org.junit.jupiter.api.Assertions.*; + +final class Xtf23HeaderTest { + private static TransferPlan plan; + + @BeforeAll + static void compileModel() throws IOException { + plan = referencePlan(); + } + + @Test + void readsHeaderAndFirstObjectFromPinnedIoxFixture() throws IOException { + var source = new SourceId("memory:/identity.xtf"); + try (var input = Files.newInputStream(fixture("identity.xtf")); + var reader = new Xtf23Reader(plan, input, Optional.of(source))) { + var start = assertInstanceOf(TransferEvent.StartTransfer.class, reader.readNext(TOKEN).orElseThrow()); + assertEquals(Optional.of("iox-reference"), start.sender()); + assertEquals(Optional.of(source), start.source()); + assertEquals(1, start.models().size()); + var model = start.models().getFirst(); + assertEquals("Reference23", plan.model(model.model()).orElseThrow().name().declared().toString()); + assertEquals(Optional.of("2026-11-06"), model.modelVersion()); + assertEquals(Optional.of("https://example.invalid/iox-reference"), model.issuer()); + var basket = assertInstanceOf(TransferEvent.StartBasket.class, reader.readNext(TOKEN).orElseThrow()); + assertEquals("records-1", basket.id().value()); + var parent = assertInstanceOf(TransferEvent.ObjectEvent.class, reader.readNext(TOKEN).orElseThrow()); + assertEquals("parent-1", parent.object().identity().tid().orElseThrow().value()); + assertEquals(Optional.of(source), parent.object().identity().source()); + assertEquals(basket.id(), parent.object().identity().currentBasket()); + } + } + + @Test + void readsEmptyAndNonemptyBasketsWithNamespacedTagsAndHeaderMetadata() { + String xml = document(MODELS + """ + + + one & two + """, """ + + + + + """).replace("", "") + .replace("", ""), + valid.replace("Reference23", "Unknown"), + valid.replace("2026-11-06", "2026-10-06"), + valid.replace("https://example.invalid/iox-reference", "https://example.invalid/other"), + valid.replace("", MODELS.substring(MODELS.indexOf("")) + + ""), + valid.replace("", ""), valid.substring(PROLOG.length()))) { + assertFailure(xml, INVALID_WIRE); + } + } + + @Test + void rejectsUnsupportedHeaderAndIncrementalBasket() { + assertFailure(document(MODELS + "", ""), UNSUPPORTED); + assertFailure(document(MODELS, ""), UNSUPPORTED); + assertFailure(document(MODELS + "", ""), INVALID_WIRE); + } + + @Test + void rejectsUnknownTopicsAndInconsistentBasketContext() { + assertFailure(document(MODELS, ""), INVALID_WIRE); + assertFailure(document(MODELS, ""), INVALID_WIRE); + try (var reader = new Xtf23Reader(plan, input(document(MODELS, + "")))) { + var failure = assertThrows(TransferProtocolException.class, () -> readAll(reader)); + assertEquals(TransferProtocolException.Reason.CONTEXT_MISMATCH, failure.reason()); + } + } + + private static void assertFailure(String xml, Xtf23Exception.Reason expected) { + try (var reader = new Xtf23Reader(plan, input(xml))) { + var failure = assertThrows(Xtf23Exception.class, () -> readAll(reader)); + assertEquals(expected, failure.reason(), failure::getMessage); + } + } +} diff --git a/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23IdentityScopeTest.java b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23IdentityScopeTest.java new file mode 100644 index 00000000..c1ee52a1 --- /dev/null +++ b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23IdentityScopeTest.java @@ -0,0 +1,69 @@ +package guru.interlis.xtf23; + +import guru.interlis.transfer.TransferEvent; +import guru.interlis.transfer.TransferPlan; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; + +import static guru.interlis.transfer.TransferIdentity.*; +import static guru.interlis.xtf23.Xtf23Exception.Reason.DUPLICATE_ID; +import static guru.interlis.xtf23.Xtf23TestData.*; +import static org.junit.jupiter.api.Assertions.*; + +final class Xtf23IdentityScopeTest { + private static TransferPlan plan; + + @BeforeAll + static void compileModel() throws IOException { + plan = referencePlan(); + } + + @Test + void detectsBothCollisionOrdersAcrossBaskets() { + for (String baskets : List.of( + """ + + + """, + """ + + + """, + """ + + """)) { + try (var reader = new Xtf23Reader(plan, input(document(MODELS, baskets)))) { + assertEquals(DUPLICATE_ID, assertThrows(Xtf23Exception.class, () -> readAll(reader)).reason()); + } + } + } + + @Test + void preservesDistinctLifetimesAndChecksTheSpellingIndependentlyOfOrigin() { + for (var basketOrigin : IdOrigin.values()) { + for (var tidOrigin : IdOrigin.values()) { + var basket = new BasketId("same", basketOrigin); + var object = new TransferId("same", tidOrigin); + var bidFirst = new Xtf23IdentityScope(); + assertTrue(bidFirst.claim(basket)); + assertFalse(bidFirst.claim(object)); + var tidFirst = new Xtf23IdentityScope(); + assertTrue(tidFirst.claim(object)); + assertFalse(tidFirst.claim(basket)); + } + } + String document = document(MODELS, """ + + + + """); + for (int i = 0; i < 2; i++) { + try (var reader = new Xtf23Reader(plan, input(document))) { + assertInstanceOf(TransferEvent.EndTransfer.class, readAll(reader).getLast()); + } + } + } +} diff --git a/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23LineAttributesTest.java b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23LineAttributesTest.java new file mode 100644 index 00000000..0955241b --- /dev/null +++ b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23LineAttributesTest.java @@ -0,0 +1,205 @@ +package guru.interlis.xtf23; + +import guru.interlis.model.SnapshotId; +import guru.interlis.model.SnapshotMismatchException; +import guru.interlis.model.SymbolRef; +import guru.interlis.transfer.TransferEvent; +import guru.interlis.transfer.TransferObject; +import guru.interlis.transfer.TransferPlan; +import guru.interlis.transfer.TransferValue; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static guru.interlis.transfer.TransferEvent.ObjectEvent; +import static guru.interlis.transfer.TransferGeometry.*; +import static guru.interlis.xtf23.Xtf23Exception.Reason.*; +import static guru.interlis.xtf23.Xtf23TestData.*; +import static org.junit.jupiter.api.Assertions.*; + +class Xtf23LineAttributesTest { + private static final String HEADER = """ + + """; + private static TransferPlan plan; + + @BeforeAll + static void compileModel() { + plan = compileInline("Lines", """ + INTERLIS 2.3; + MODEL Lines (en) AT "https://example.invalid/lines" VERSION "1" = + DOMAIN Point = COORD 0.00..100.00, 0.00..100.00; + TOPIC Data = + STRUCTURE Details = + note : TEXT*20; + END Details; + STRUCTURE Marks = + label : MANDATORY TEXT*20; + details : BAG {0..*} OF Details; + END Marks; + STRUCTURE Other = + label : MANDATORY TEXT*20; + END Other; + CLASS Item = + trace : POLYLINE WITH (STRAIGHTS) VERTEX Point; + shape : SURFACE WITH (STRAIGHTS) VERTEX Point WITHOUT OVERLAPS > 0.01 LINE ATTRIBUTES Marks; + parcel : AREA WITH (STRAIGHTS) VERTEX Point WITHOUT OVERLAPS > 0.01 LINE ATTRIBUTES Marks; + END Item; + END Data; + END Lines. + """); + } + + @Test + void roundTripsAttributedSurfaceAndAreaPartitionsAndPlainPolyline() { + String east = line(marks("east").replace("", """ +
inside
+ """), point(0, 0), point(10, 0), point(10, 10)); + String west = line(marks("west"), point(10, 10), point(0, 10), point(0, 0)); + String inner = line(marks("hole"), point(2, 2), point(2, 4), point(4, 4), point(2, 2)); + String shape = "" + east + west + "" + inner + + ""; + String area = "" + east + west + ""; + String trace = "" + line("", point(0, 0), point(1, 1)) + ""; + var events = read(transfer(trace + shape + area)); + var object = ((ObjectEvent) events.get(2)).object(); + assertTrue(assertInstanceOf(Polyline.class, field(object, "trace")).lineAttributes().isEmpty()); + for (String field : List.of("shape", "parcel")) { + var surface = assertInstanceOf(Surface.class, field(object, field)); + assertEquals(List.of("east", "west"), surface.shell().lines().stream() + .map(Xtf23LineAttributesTest::label).toList()); + } + var surface = assertInstanceOf(Surface.class, field(object, "shape")); + var attributes = surface.shell().lines().getFirst().lineAttributes().orElseThrow(); + var details = assertInstanceOf(TransferValue.Repeated.class, + attributes.value(slot(type("Marks"), "details")).orElseThrow()); + var detail = assertInstanceOf(TransferValue.Structured.class, details.values().getFirst()); + assertEquals(type("Details"), detail.type()); + assertEquals(new TransferValue.Text("inside"), detail.value(slot(type("Details"), "note")).orElseThrow()); + assertEquals("hole", label(surface.holes().getFirst().lines().getFirst())); + String wire = write(events); + assertTrue(wire.contains("")); + assertTrue(wire.contains("")); + assertEquals(events, read(wire)); + } + + @Test + void rejectsMissingUnexpectedMisorderedDuplicateAndUnrelatedLineStructures() { + String first = point(0, 0); + String end = point(1, 1); + String mark = marks("first"); + String complete = line(mark, first, end, first); + for (String lines : List.of( + line("", first, end), + complete + line("", first, end), + line(mark.replace(".Marks", ".Other"), first, end), + line(mark.replace(".Marks", ".Item"), first, end), + line("", first, end), + line(mark + mark, first, end), + line(mark.replace("", ""), first, end), + "" + first + mark + end + "", + "" + mark + first + end + mark + "")) { + assertEquals(INVALID_WIRE, assertThrows(Xtf23Exception.class, + () -> read(transfer("" + lines + "")), + lines).reason()); + } + assertEquals(INVALID_WIRE, assertThrows(Xtf23Exception.class, + () -> read(transfer("" + complete + ""))).reason()); + assertEquals(UNSUPPORTED, assertThrows(Xtf23Exception.class, () -> read(transfer( + "" + mark + "" + first + end + + ""))).reason()); + } + + @Test + void validatesWriterLineStructureAndRejectsForeignSnapshotBeforeWriting() { + String line = line(marks("valid"), point(0, 0), point(1, 1), point(0, 0)); + var events = read(transfer("" + line + "")); + var object = ((ObjectEvent) events.get(2)).object(); + var original = assertInstanceOf(Surface.class, field(object, "shape")).shell().lines().getFirst(); + var absent = new Polyline(original.parts()); + var wrong = new Polyline(original.parts(), + Optional.of(new TransferValue.Structured(type("Other"), Map.of()))); + for (var bad : List.of(absent, wrong)) { + assertEquals(INVALID_WIRE, assertThrows(Xtf23Exception.class, () -> write(withField(events, "shape", + new Surface(new Ring(List.of(bad)), List.of())))).reason()); + } + assertEquals(INVALID_WIRE, assertThrows(Xtf23Exception.class, + () -> write(withField(events, "trace", original))).reason()); + + var foreign = new SymbolRef(new SnapshotId("foreign-line-model"), type("Marks").symbol()); + var foreignLine = new Polyline(original.parts(), + Optional.of(new TransferValue.Structured(foreign, Map.of()))); + assertThrows(SnapshotMismatchException.class, () -> withField(events, "shape", + new Surface(new Ring(List.of(original)), List.of(new Ring(List.of(foreignLine)))))); + } + + private static String label(Polyline line) { + var structure = line.lineAttributes().orElseThrow(); + assertEquals(type("Marks"), structure.type()); + var label = (TransferValue.Text) structure.value(slot(structure.type(), "label")).orElseThrow(); + return label.value(); + } + + private static String marks(String value) { + return ""; + } + + private static String point(int x, int y) { + return "" + x + ".00" + y + ".00"; + } + + private static String line(String attrs, String... points) { + return "" + attrs + String.join("", points) + ""; + } + + private static String transfer(String fields) { + return document(HEADER, "" + + fields + ""); + } + + private static List read(String source) { + try (var reader = new Xtf23Reader(plan, input(source))) { + return readAll(reader); + } + } + + private static String write(List events) { + var output = new ByteArrayOutputStream(); + try (var writer = new Xtf23Writer(plan, output)) { + for (var event : events) { + writer.write(event, TOKEN); + } + } + return output.toString(StandardCharsets.UTF_8); + } + + private static TransferValue field(TransferObject object, String name) { + return object.value(slot(object.type(), name)).orElseThrow(); + } + + private static List withField(List events, String name, TransferValue value) { + var replacement = new ArrayList<>(events); + var event = (ObjectEvent) events.get(2); + var object = event.object(); + replacement.set(2, new ObjectEvent(new TransferObject(object.identity(), + new TransferValue.Structured(object.type(), Map.of(slot(object.type(), name), value))), + event.operation(), event.consistency())); + return replacement; + } + + private static SymbolRef type(String name) { + return plan.objects().stream().filter(type -> type.name().declared().last().equals(name)) + .findFirst().orElseThrow().symbol(); + } + + private static SymbolRef slot(SymbolRef type, String name) { + return plan.object(type).orElseThrow().slots().stream() + .filter(slot -> slot.name().declared().last().equals(name)).findFirst().orElseThrow().symbol(); + } +} diff --git a/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23ObjectEncodingTest.java b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23ObjectEncodingTest.java new file mode 100644 index 00000000..9adae523 --- /dev/null +++ b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23ObjectEncodingTest.java @@ -0,0 +1,235 @@ +package guru.interlis.xtf23; + +import guru.interlis.model.CollectionKind; +import guru.interlis.model.SymbolRef; +import guru.interlis.transfer.TransferEvent.ObjectEvent; +import guru.interlis.transfer.TransferPlan; +import guru.interlis.transfer.TransferValue; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.file.Files; +import java.util.List; +import java.util.Optional; + +import static guru.interlis.transfer.TransferIdentity.*; +import static guru.interlis.transfer.TransferValue.*; +import static guru.interlis.xtf23.Xtf23Exception.Reason.*; +import static guru.interlis.xtf23.Xtf23TestData.*; +import static org.junit.jupiter.api.Assertions.*; + +final class Xtf23ObjectEncodingTest { + private static TransferPlan reference; + private static TransferPlan values; + private static final String VALUES = """ + + """; + private static final String OIDSPACES = """ + + """; + + @BeforeAll + static void compileModels() throws IOException { + reference = referencePlan(); + values = compileInline("Values", """ + INTERLIS 2.3; + MODEL Values (en) AT "https://example.invalid/values" VERSION "1" = + TOPIC Data = + CLASS Parent = END Parent; + STRUCTURE Detail = + label: TEXT*32; + owner: REFERENCE TO (EXTERNAL) Parent; + END Detail; + CLASS Item = + quantity: -100.000 .. 999.000; + flag: BOOLEAN; + shade: (dark (red, blue), light); + label: TEXT*32; + notes: MTEXT*64; + one: Detail; + details: BAG {0..*} OF Detail; + sequence: LIST {0..*} OF Detail; + END Item; + ASSOCIATION Note = + parent (EXTERNAL) -- {0..1} Parent; + item -- {0..*} Item; + remark: TEXT*32; + END Note; + ASSOCIATION Peer = + left (ORDERED, EXTERNAL) -- {0..*} Parent; + right -- {0..*} Item; + remark: TEXT*32; + END Peer; + ASSOCIATION Stable (OID) = + first -- {0..1} Parent; + second -- {0..*} Item; + END Stable; + END Data; + END Values. + """); + } + + @Test + void readsPinnedObjectsAndDoesNotEvaluateUniqueAcrossBaskets() throws IOException { + for (String fixture : List.of("identity.xtf", "unique-across-baskets.xtf")) { + try (var source = Files.newInputStream(fixture(fixture)); + var reader = new Xtf23Reader(reference, source)) { + var objects = objects(reader); + assertEquals(3, objects.size()); + var item = objects.get(1); + assertEquals(new Text("same"), field(reference, item, "code")); + var details = assertInstanceOf(Repeated.class, field(reference, item, "details")); + assertEquals(CollectionKind.LIST, details.kind()); + assertEquals(List.of(new Text("zéro & \\"), new Text("alpha")), + details.values().stream().map(value -> field(reference, (Structured) value, "description")).toList()); + var parent = assertInstanceOf(Reference.class, + item.value(slot(reference, "Reference23.Records.ParentLink", "parent")).orElseThrow()); + assertEquals(new TransferId("parent-1", IdOrigin.SOURCE), parent.target()); + assertEquals(Optional.of(new BasketId("records-1", IdOrigin.SOURCE)), parent.targetBasket()); + assertFalse(parent.external()); + assertEquals(new Text(fixture.equals("identity.xtf") ? "other" : "same"), + field(reference, objects.getLast(), "code")); + } + } + } + + @Test + void keepsScalarLexemesEmptyTextAndOrderedStructureValues() { + String detail = ""; + String data = """ + +1.2300e+2truedark.red +