From 9d250a550309963ae40f173a4755eb0912654567 Mon Sep 17 00:00:00 2001 From: Stefan Ziegler Date: Fri, 6 Nov 2026 21:34:06 +0100 Subject: [PATCH 1/6] Stream XTF 2.3 document headers and baskets --- build/interlis/JvmBuildInputs.java | 2 +- build/interlis/ModulePathContract.java | 15 +- docs/architecture.md | 3 +- docs/development/build.md | 2 +- .../build.jenesis/test-inputs.properties | 2 + interlis-xtf23/pom.xml | 42 ++ .../guru/interlis/xtf23/Xtf23Exception.java | 35 ++ .../java/guru/interlis/xtf23/Xtf23Input.java | 225 ++++++++++ .../guru/interlis/xtf23/Xtf23OidSpace.java | 25 ++ .../java/guru/interlis/xtf23/Xtf23Reader.java | 392 ++++++++++++++++++ interlis-xtf23/src/main/java/module-info.java | 8 + .../guru/interlis/xtf23/Xtf23HeaderTest.java | 130 ++++++ .../xtf23/Xtf23StreamLifecycleTest.java | 155 +++++++ .../guru/interlis/xtf23/Xtf23TestData.java | 79 ++++ pom.xml | 1 + 15 files changed, 1110 insertions(+), 6 deletions(-) create mode 100644 interlis-xtf23/build.jenesis/test-inputs.properties create mode 100644 interlis-xtf23/pom.xml create mode 100644 interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Exception.java create mode 100644 interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Input.java create mode 100644 interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23OidSpace.java create mode 100644 interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Reader.java create mode 100644 interlis-xtf23/src/main/java/module-info.java create mode 100644 interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23HeaderTest.java create mode 100644 interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23StreamLifecycleTest.java create mode 100644 interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23TestData.java diff --git a/build/interlis/JvmBuildInputs.java b/build/interlis/JvmBuildInputs.java index 8ea27d08..49d1097d 100644 --- a/build/interlis/JvmBuildInputs.java +++ b/build/interlis/JvmBuildInputs.java @@ -12,7 +12,7 @@ public final class JvmBuildInputs { public static final List MODULES = List.of("interlis-model", "interlis-compiler", "interlis-workspace", "interlis-repository", "interlis-projection", "interlis-documentation", - "interlis-transfer", "interlis-wasm-bridge", "interlis-compiler-cli", "interlis-lsp", + "interlis-transfer", "interlis-xtf23", "interlis-wasm-bridge", "interlis-compiler-cli", "interlis-lsp", "interlis-compiler-gui", "interlis-verification"); public static void requireProject(Path root) throws Exception { diff --git a/build/interlis/ModulePathContract.java b/build/interlis/ModulePathContract.java index 93228025..9f2c1728 100644 --- a/build/interlis/ModulePathContract.java +++ b/build/interlis/ModulePathContract.java @@ -15,6 +15,13 @@ public static String name(String project) { public static void requireOwnDescriptor(String project, ModuleDescriptor descriptor) { if(descriptor.isAutomatic()||descriptor.isOpen()||!descriptor.name().equals(name(project))) throw new IllegalStateException("Own runtime module must be explicit and encapsulated: "+project); + if(project.equals("interlis-xtf23")) { + var required=descriptor.requires().stream().map(ModuleDescriptor.Requires::name).collect(java.util.stream.Collectors.toSet()); + if(!required.equals(Set.of("java.base","java.xml","guru.interlis.model","guru.interlis.transfer")) + ||descriptor.exports().size()!=1||descriptor.exports().stream().anyMatch(export-> + export.isQualified()||!export.source().equals("guru.interlis.xtf23"))||!descriptor.opens().isEmpty()) + throw new IllegalStateException("Unexpected XTF 2.3 module boundary: "+descriptor); + } } public static void run(Path root)throws Exception { for(String project:JvmBuildInputs.MODULES)if(!Files.isRegularFile(root.resolve(project+"/src/main/java/module-info.java")))throw new IllegalStateException("Missing own module descriptor: "+project); @@ -26,11 +33,12 @@ public static void run(Path root)throws Exception { if(!result.explicitDescriptor()||!result.moduleName().orElseThrow().equals(name(project)))throw new IllegalStateException("Wrong own module: "+project); own.add(name(project)); var runtime=ModuleArtifacts.runtime(root,project,outcome); - if(project.equals("interlis-transfer")) { + if(project.equals("interlis-transfer")||project.equals("interlis-xtf23")) { String prefix="maven/guru.interlis/", version="/"+JvmBuildInputs.projectVersion(root); - var expected=Set.of(prefix+"interlis-transfer"+version,prefix+"interlis-model"+version); + var expected=new HashSet<>(Set.of(prefix+"interlis-transfer"+version,prefix+"interlis-model"+version)); + if(project.equals("interlis-xtf23"))expected.add(prefix+project+version); var actual=runtime.stream().map(ModuleArtifacts.Artifact::coordinates).collect(java.util.stream.Collectors.toSet()); - if(!actual.equals(expected))throw new IllegalStateException("Unexpected transfer runtime closure: "+actual); + if(!actual.equals(expected))throw new IllegalStateException("Unexpected runtime closure for "+project+": "+actual); } for(var artifact:runtime) { Path prior=files.putIfAbsent(artifact.fileName(),artifact.file()); @@ -71,6 +79,7 @@ public static void run(Path root)throws Exception { "guru.interlis.projection","guru.interlis.projection.DiagramProjector", "guru.interlis.documentation","guru.interlis.documentation.DocumentationProjector", "guru.interlis.transfer","guru.interlis.transfer.TransferPlanner", + "guru.interlis.xtf23","guru.interlis.xtf23.Xtf23Reader", "guru.interlis.wasm.bridge","guru.interlis.bridge.BridgeDispatcher"); String extra=String.join("\n",own.stream().filter(entrypoints::containsKey).map(name-> "if (named && !"+entrypoints.get(name)+".class.getModule().getName().equals(\""+name+"\")) throw new AssertionError(\"Wrong public entrypoint module\");").toList()); diff --git a/docs/architecture.md b/docs/architecture.md index 7eef1cfd..94e654c1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -21,13 +21,14 @@ Die zugehörigen POMs beschreiben die konkreten Abhängigkeiten und Scopes. | `interlis-projection` | compiler, model | Diagramm-DTOs und reine Projektion | | `interlis-documentation` | model, projection | Lesende Dokumentationsprojektion und Rendering | | `interlis-transfer` | model | Formatunabhängige Transferpläne, Objektwerte und Streaming-Protokoll | +| `interlis-xtf23` | transfer, model | XML-Adapter für INTERLIS 2.3 | | `interlis-compiler-cli` | compiler, model, repository, workspace, projection | CLI, Prozessgrenzen, Präsentation und Ausgabe | | `interlis-compiler-gui` | model, compiler, workspace, repository | Desktop-Modellprüfung, Diagnosen, Abhängigkeiten und Repositoryexplorer | | `interlis-lsp` | workspace, compiler, projection, model, repository, documentation | LSP-/Editorhost, Scheduling und Ausgaben | | `interlis-wasm-bridge` | workspace, compiler, projection, model | Gemeinsames JSON-Protokoll und Sitzungen | | `interlis-verification` | compiler, model, workspace, repository, wasm-bridge | Prüfwerkzeuge ausserhalb der SDK-Auslieferung | -Alle zwölf JVM-Module werden über denselben Builder erzeugt. Die +Alle dreizehn JVM-Module werden über denselben Builder erzeugt. Die jeweils ausgewählten JPMS-Closures stehen im [Buildvertrag](development/build.md). TeaVM und Web Image konsumieren denselben expliziten JVM-Snapshot. Browser und Editor teilen den neutralen diff --git a/docs/development/build.md b/docs/development/build.md index 0e29fd6b..d2cd5ee3 100644 --- a/docs/development/build.md +++ b/docs/development/build.md @@ -1,6 +1,6 @@ # Build und Toolchains -Jenesis 0.18.1 erzeugt die zwölf JVM-Module mit Java 25 ohne Preview. Die POMs +Jenesis 0.18.1 erzeugt die dreizehn JVM-Module mit Java 25 ohne Preview. Die POMs definieren Group, Artifact, Version, Scopes und Pins. Der Upstreampin gehört zu den [Jenesis-Quellen](../../build/JENESIS-UPSTREAM.json). Der gepinnte Gradle-Wrapper baut ausschliesslich TeaVM und GraalVM Web Image. diff --git a/interlis-xtf23/build.jenesis/test-inputs.properties b/interlis-xtf23/build.jenesis/test-inputs.properties new file mode 100644 index 00000000..120c2530 --- /dev/null +++ b/interlis-xtf23/build.jenesis/test-inputs.properties @@ -0,0 +1,2 @@ +input.0=testdata/reference/iox/ili23/Reference23.ili +input.1=testdata/reference/iox/ili23/identity.xtf diff --git a/interlis-xtf23/pom.xml b/interlis-xtf23/pom.xml new file mode 100644 index 00000000..37b300f5 --- /dev/null +++ b/interlis-xtf23/pom.xml @@ -0,0 +1,42 @@ + + + 4.0.0 + guru.interlisinterlis-java0.2.0-SNAPSHOT + interlis-xtf23 + + guru.interlisinterlis-transfer${project.version} + guru.interlisinterlis-model${project.version} + guru.interlisinterlis-compiler${project.version}test + org.junit.jupiterjunit-jupitertest + org.junit.platformjunit-platform-launchertest + + + diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Exception.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Exception.java new file mode 100644 index 00000000..85dc813a --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Exception.java @@ -0,0 +1,35 @@ +package guru.interlis.xtf23; + +/** An XML, wire format, capability or I/O failure at the XTF 2.3 boundary. */ +public final class Xtf23Exception extends IllegalStateException { + /** Distinguishes format failures from an unsupported feature or host failure. */ + public enum Reason { + MALFORMED_XML, UNSAFE_XML, INVALID_WIRE, UNSUPPORTED, LIMIT_EXCEEDED, IO, CONFIGURATION + } + + private final Reason reason; + private final int line; + private final int column; + + Xtf23Exception(Reason reason, String message, int line, int column, Throwable cause) { + super(line > 0 ? message + " (line " + line + ", column " + column + ")" : message, cause); + this.reason = reason; + this.line = line; + this.column = column; + } + + /** Returns the stable failure category. */ + public Reason reason() { + return reason; + } + + /** Returns the one-based XML line, or -1 when unavailable. */ + public int line() { + return line; + } + + /** Returns the one-based XML column, or -1 when unavailable. */ + public int column() { + return column; + } +} diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Input.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Input.java new file mode 100644 index 00000000..6178c646 --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Input.java @@ -0,0 +1,225 @@ +package guru.interlis.xtf23; + +import guru.interlis.execution.CancellationToken; +import guru.interlis.execution.OperationCancelledException; + +import javax.xml.XMLConstants; +import javax.xml.stream.Location; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; + +import static guru.interlis.xtf23.Xtf23Exception.Reason.*; +import static javax.xml.stream.XMLStreamConstants.*; + +/** Secure cursor over the XTF XML stream. */ +final class Xtf23Input { + static final int MAX_TEXT = 1_048_576; + static final int MAX_TOKEN_BYTES = 8 * MAX_TEXT; + private static final int MAX_DEPTH = 128; + private static final int MAX_ATTRIBUTES = 64; + private static final int MAX_NAME = 1024; + + private final BorrowedInput input; + private XMLStreamReader xml; + + Xtf23Input(InputStream input) { + this.input = new BorrowedInput(Objects.requireNonNull(input, "input")); + } + + void begin(CancellationToken cancellation) { + input.cancellation = cancellation; + } + + void finish() { + input.cancellation = CancellationToken.none(); + } + + void open() throws XMLStreamException { + var factory = XMLInputFactory.newDefaultFactory(); + try { + requireProperty(factory, XMLInputFactory.IS_NAMESPACE_AWARE, true); + requireProperty(factory, XMLInputFactory.IS_COALESCING, false); + requireProperty(factory, XMLInputFactory.IS_VALIDATING, false); + requireProperty(factory, XMLInputFactory.SUPPORT_DTD, false); + requireProperty(factory, XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + requireProperty(factory, XMLConstants.ACCESS_EXTERNAL_DTD, ""); + requireProperty(factory, "jdk.xml.maxElementDepth", MAX_DEPTH); + requireProperty(factory, "jdk.xml.elementAttributeLimit", MAX_ATTRIBUTES); + requireProperty(factory, "jdk.xml.maxXMLNameLimit", MAX_NAME); + } catch (IllegalArgumentException e) { + throw failure(CONFIGURATION, "Cannot configure the JDK XML input factory", null, e); + } + factory.setXMLResolver((publicId, systemId, baseUri, namespace) -> { + throw error(UNSAFE_XML, "External XML resolution is forbidden"); + }); + input.remaining = MAX_TOKEN_BYTES; + xml = factory.createXMLStreamReader(input); + if (!"1.0".equals(xml.getVersion()) || !"UTF-8".equalsIgnoreCase(xml.getCharacterEncodingScheme()) + || !"UTF-8".equalsIgnoreCase(xml.getEncoding())) { + throw error(INVALID_WIRE, "XTF 2.3 requires an XML 1.0 declaration and UTF-8 encoding"); + } + } + + private static void requireProperty(XMLInputFactory factory, String property, Object value) { + factory.setProperty(property, value); + if (!value.toString().equals(Objects.toString(factory.getProperty(property), null))) { + throw new IllegalArgumentException("JDK XML property did not take effect: " + property); + } + } + + XMLStreamReader cursor() { + return xml; + } + + int next() throws XMLStreamException { + input.cancellation.throwIfCancellationRequested(); + input.remaining = MAX_TOKEN_BYTES; + int token = xml.next(); + if (token == DTD || token == ENTITY_REFERENCE) { + throw error(UNSAFE_XML, "DTD declarations and custom XML entities are forbidden"); + } + return token; + } + + int nextTag() throws XMLStreamException { + while (true) { + int token = next(); + switch (token) { + case START_ELEMENT, END_ELEMENT, END_DOCUMENT -> { return token; } + case COMMENT, PROCESSING_INSTRUCTION -> { } + case CHARACTERS, CDATA, SPACE -> { + if (!xml.isWhiteSpace()) { + throw error(INVALID_WIRE, "Unexpected text between XTF elements"); + } + } + default -> throw error(INVALID_WIRE, "Unexpected XML event " + token); + } + } + } + + String text() throws XMLStreamException { + var value = new StringBuilder(); + while (true) { + int token = next(); + switch (token) { + case END_ELEMENT -> { return value.toString(); } + case COMMENT, PROCESSING_INSTRUCTION -> { } + case CHARACTERS, CDATA, SPACE -> { + if (xml.getTextLength() > MAX_TEXT - value.length()) { + throw error(LIMIT_EXCEEDED, "XTF text value exceeds " + MAX_TEXT + " UTF-16 units"); + } + value.append(xml.getTextCharacters(), xml.getTextStart(), xml.getTextLength()); + } + default -> throw error(INVALID_WIRE, "Expected text without child elements"); + } + } + } + + void close() { + if (xml != null) { + try { + xml.close(); + } catch (XMLStreamException e) { + throw translate(e); + } finally { + xml = null; + } + } + } + + Xtf23Exception error(Xtf23Exception.Reason reason, String message) { + return failure(reason, message, xml == null ? null : xml.getLocation(), null); + } + + Xtf23Exception translate(XMLStreamException e) { + Throwable nested = e.getNestedException(); + if (nested instanceof Xtf23Exception failure) { + return failure; + } + if (nested instanceof OperationCancelledException cancelled) { + throw cancelled; + } + String message = e.getMessage(); + if (nested instanceof TokenLimit || message != null && message.contains("JAXP0001")) { + return failure(LIMIT_EXCEEDED, "XML structure or token exceeds the XTF reader limits", e.getLocation(), e); + } + if (nested instanceof SourceFailure) { + return failure(IO, "Cannot read the XTF input stream", e.getLocation(), e); + } + return failure(MALFORMED_XML, "Malformed XML: " + message, e.getLocation(), e); + } + + private static Xtf23Exception failure(Xtf23Exception.Reason reason, String message, + Location location, Throwable cause) { + return new Xtf23Exception(reason, message, location == null ? -1 : location.getLineNumber(), + location == null ? -1 : location.getColumnNumber(), cause); + } + + private static final class TokenLimit extends IOException { } + + private static final class SourceFailure extends IOException { + SourceFailure(IOException cause) { + super(cause); + } + } + + /** StAX can close the source at EOF. Limit parser reads within an individual XML advance. */ + private static final class BorrowedInput extends FilterInputStream { + private CancellationToken cancellation = CancellationToken.none(); + private int remaining; + + BorrowedInput(InputStream input) { + super(input); + } + + private void requireBudget() throws TokenLimit { + cancellation.throwIfCancellationRequested(); + if (remaining == 0) { + throw new TokenLimit(); + } + } + + @Override + public int read() throws IOException { + requireBudget(); + int value; + try { + value = in.read(); + } catch (IOException failure) { + throw new SourceFailure(failure); + } + if (value != -1) { + remaining--; + } + return value; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + if (length == 0) { + return 0; + } + requireBudget(); + int count; + try { + count = in.read(buffer, offset, Math.min(length, remaining)); + } catch (IOException failure) { + throw new SourceFailure(failure); + } + if (count > 0) { + remaining -= count; + } + return count; + } + + @Override + public void close() { + // The caller owns the source even when StAX reaches EOF or abandons parsing. + } + } +} diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23OidSpace.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23OidSpace.java new file mode 100644 index 00000000..2ea8971d --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23OidSpace.java @@ -0,0 +1,25 @@ +package guru.interlis.xtf23; + +import java.util.Objects; +import java.util.regex.Pattern; + +/** + * One XTF 2.3 header declaration connecting an XML-ID prefix to its OID domain. + * Both components retain their exact source spelling. + */ +public record Xtf23OidSpace(String alias, String domain) { + static final String NC_NAME = "[A-Za-z_][A-Za-z0-9_.-]*"; + private static final Pattern ALIAS = Pattern.compile(NC_NAME); + private static final Pattern QUALIFIED_DOMAIN = + Pattern.compile("[A-Za-z][A-Za-z0-9_]*(?:\\.[A-Za-z][A-Za-z0-9_]*)+"); + + /** Creates a declaration with a nonnull XML prefix and qualified INTERLIS domain name. */ + public Xtf23OidSpace { + Objects.requireNonNull(alias, "alias"); + Objects.requireNonNull(domain, "domain"); + if (alias.length() > 1024 || domain.length() > 1024 || !ALIAS.matcher(alias).matches() + || !QUALIFIED_DOMAIN.matcher(domain).matches()) { + throw new IllegalArgumentException("Invalid OIDSPACE alias or qualified domain name"); + } + } +} diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Reader.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Reader.java new file mode 100644 index 00000000..e022a127 --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Reader.java @@ -0,0 +1,392 @@ +package guru.interlis.xtf23; + +import guru.interlis.execution.CancellationToken; +import guru.interlis.model.LanguageVersion; +import guru.interlis.model.ModelSymbol; +import guru.interlis.model.SymbolKind; +import guru.interlis.model.SymbolRef; +import guru.interlis.source.SourceId; +import guru.interlis.transfer.ModelDescriptor; +import guru.interlis.transfer.ObjectDescriptor; +import guru.interlis.transfer.TopicDescriptor; +import guru.interlis.transfer.TransferEvent; +import guru.interlis.transfer.TransferIdentity; +import guru.interlis.transfer.TransferName; +import guru.interlis.transfer.TransferObject; +import guru.interlis.transfer.TransferPlan; +import guru.interlis.transfer.TransferReader; +import guru.interlis.transfer.TransferValue; + +import javax.xml.XMLConstants; +import javax.xml.stream.XMLStreamException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static guru.interlis.transfer.TransferEvent.*; +import static guru.interlis.transfer.TransferIdentity.*; +import static guru.interlis.xtf23.Xtf23Exception.Reason.*; +import static javax.xml.stream.XMLStreamConstants.*; + +/** + * Pulls one XTF 2.3 FULL transfer from a caller-owned byte stream. Constructing this reader + * performs no input. Close releases the parser without closing the supplied stream. + * Events are immutable and can be retained after the reader advances. + * The XML reader bounds depth to 128 elements, scalar text to 1,048,576 UTF-16 + * code units, and input read during any individual parser advance to 8 MiB. + */ +public final class Xtf23Reader extends TransferReader { + static final String NAMESPACE = "http://www.interlis.ch/INTERLIS2.3"; + static final Pattern XML_ID = Pattern.compile("(?:" + Xtf23OidSpace.NC_NAME + ":)?[A-Za-z0-9_][A-Za-z0-9_.-]*"); + private static final String VERSION = "2.3"; + private static final int MAX_OID_SPACES = 4096; + private static final int MAX_IDENTIFIER = 4096; + + private final TransferPlan plan; + private final Xtf23Input xml; + private final Optional source; + private final Map models; + private final Map topics; + private final Map objects; + private Map oidSpaces = Map.of(); + private List oidSpaceDeclarations = List.of(); + private State state = State.BEFORE_TRANSFER; + private BasketId basket; + + /** Creates a reader over the nonnull plan and caller-owned input. */ + public Xtf23Reader(TransferPlan plan, InputStream input) { + this(plan, input, Optional.empty()); + } + + /** Creates a reader with an optional nonnull opaque source identity for events. */ + public Xtf23Reader(TransferPlan plan, InputStream input, Optional source) { + super(plan); + this.plan = plan; + this.source = Objects.requireNonNull(source, "source"); + xml = new Xtf23Input(input); + models = index(plan.models(), ModelDescriptor::name); + topics = index(plan.topics(), TopicDescriptor::name); + objects = index(plan.objects(), ObjectDescriptor::name); + } + + /** Returns the immutable, ordered OIDSPACE declarations after reading the header. + * An empty list represents no declarations, including before the first read. */ + public List oidSpaces() { + return oidSpaceDeclarations; + } + + private static Map index(List descriptors, Function names) { + return descriptors.stream().collect(Collectors.toUnmodifiableMap( + value -> names.apply(value).declared().toString(), Function.identity())); + } + + @Override + protected Optional readEvent(CancellationToken cancellation) { + xml.begin(cancellation); + try { + return switch (state) { + case BEFORE_TRANSFER -> Optional.of(startTransfer()); + case BETWEEN_BASKETS -> Optional.of(basketOrEndTransfer()); + case IN_BASKET -> Optional.of(objectOrEndBasket()); + case DONE -> Optional.empty(); + }; + } catch (XMLStreamException failure) { + throw xml.translate(failure); + } finally { + xml.finish(); + } + } + + @Override + protected void release() { + basket = null; + oidSpaces = Map.of(); + xml.close(); + } + + private StartTransfer startTransfer() throws XMLStreamException { + xml.open(); + require(START_ELEMENT, xml.nextTag(), "TRANSFER"); + attributes(true); + require(START_ELEMENT, xml.nextTag(), "HEADERSECTION"); + var header = attributes(false, "VERSION", "SENDER"); + if (!VERSION.equals(required(header, "VERSION"))) { + throw xml.error(INVALID_WIRE, "Expected XTF header VERSION " + VERSION); + } + String sender = required(header, "SENDER"); + require(START_ELEMENT, xml.nextTag(), "MODELS"); + attributes(false); + var declarations = readModels(); + + int token = xml.nextTag(); + if (is(START_ELEMENT, token, "ALIAS")) { + attributes(false); + if (!is(END_ELEMENT, xml.nextTag(), "ALIAS")) { + throw xml.error(UNSUPPORTED, "Nonempty polymorphic ALIAS tables are unsupported"); + } + token = xml.nextTag(); + } + Map spaces = Map.of(); + if (is(START_ELEMENT, token, "OIDSPACES")) { + spaces = readOidSpaces(); + token = xml.nextTag(); + } + Optional comment = Optional.empty(); + if (is(START_ELEMENT, token, "COMMENT")) { + attributes(false); + comment = Optional.of(xml.text()); + token = xml.nextTag(); + } + require(END_ELEMENT, token, "HEADERSECTION"); + require(START_ELEMENT, xml.nextTag(), "DATASECTION"); + attributes(false); + oidSpaces = spaces; + oidSpaceDeclarations = List.copyOf(spaces.values()); + state = State.BETWEEN_BASKETS; + return new StartTransfer(source, Optional.of(sender), comment, declarations); + } + + private List readModels() throws XMLStreamException { + var declarations = new ArrayList(); + var declared = new HashSet(); + int token = xml.nextTag(); + while (!is(END_ELEMENT, token, "MODELS")) { + require(START_ELEMENT, token, "MODEL"); + var attributes = attributes(false, "NAME", "VERSION", "URI"); + String name = required(attributes, "NAME"); + String version = required(attributes, "VERSION"); + String issuer = required(attributes, "URI"); + var model = models.get(name); + if (model == null) { + throw xml.error(INVALID_WIRE, "Unknown XTF model " + name); + } + if (model.languageVersion() != LanguageVersion.ILI23) { + throw xml.error(UNSUPPORTED, "Model " + name + " does not use INTERLIS 2.3"); + } + var symbol = (ModelSymbol) plan.symbol(model.symbol()).known().orElseThrow(); + if (!version.equals(model.modelVersion()) || !issuer.equals(symbol.issuer())) { + throw xml.error(INVALID_WIRE, "XTF declaration conflicts with model " + name); + } + if (!declared.add(model.symbol())) { + throw xml.error(INVALID_WIRE, "Repeated XTF model declaration " + name); + } + require(END_ELEMENT, xml.nextTag(), "MODEL"); + declarations.add(new ModelDeclaration(model.symbol(), model.languageVersion(), + Optional.of(version), Optional.of(issuer))); + token = xml.nextTag(); + } + if (declarations.isEmpty()) { + throw xml.error(INVALID_WIRE, "MODELS requires at least one MODEL"); + } + return declarations; + } + + private Map readOidSpaces() throws XMLStreamException { + attributes(false); + var spaces = new LinkedHashMap(); + int token = xml.nextTag(); + while (!is(END_ELEMENT, token, "OIDSPACES")) { + if (spaces.size() == MAX_OID_SPACES) { + throw xml.error(LIMIT_EXCEEDED, "Too many XTF OIDSPACE declarations"); + } + require(START_ELEMENT, token, "OIDSPACE"); + var attributes = attributes(false, "NAME", "OIDDOMAIN"); + String alias = required(attributes, "NAME"); + String domain = required(attributes, "OIDDOMAIN"); + Xtf23OidSpace space; + try { + space = new Xtf23OidSpace(alias, domain); + } catch (IllegalArgumentException e) { + throw xml.error(INVALID_WIRE, e.getMessage()); + } + if (spaces.putIfAbsent(alias, space) != null) { + throw xml.error(INVALID_WIRE, "Repeated OIDSPACE alias " + alias); + } + require(END_ELEMENT, xml.nextTag(), "OIDSPACE"); + token = xml.nextTag(); + } + if (spaces.isEmpty()) { + throw xml.error(INVALID_WIRE, "OIDSPACES requires at least one OIDSPACE"); + } + return spaces; + } + + private TransferEvent basketOrEndTransfer() throws XMLStreamException { + int token = xml.nextTag(); + if (is(END_ELEMENT, token, "DATASECTION")) { + require(END_ELEMENT, xml.nextTag(), "TRANSFER"); + if (xml.nextTag() != END_DOCUMENT) { + throw xml.error(INVALID_WIRE, "Unexpected XML after the XTF transfer"); + } + state = State.DONE; + oidSpaces = Map.of(); + return new EndTransfer(); + } + String tag = elementName(token); + var topic = topic(tag); + var attributes = attributes(false, "BID", "TOPICS", "KIND", "STARTSTATE", "ENDSTATE", "CONSISTENCY"); + switch (attributes.getOrDefault("KIND", "FULL")) { + case "FULL" -> { } + case "INITIAL", "UPDATE" -> throw xml.error(UNSUPPORTED, "Incremental XTF baskets are unsupported"); + default -> throw xml.error(INVALID_WIRE, "Unknown XTF basket KIND"); + } + if (attributes.containsKey("STARTSTATE") || attributes.containsKey("ENDSTATE")) { + throw xml.error(INVALID_WIRE, "A FULL basket cannot carry incremental state"); + } + var id = basketId(required(attributes, "BID")); + var extensions = new ArrayList(); + if (attributes.containsKey("TOPICS")) { + var seen = new HashSet(); + for (String name : attributes.get("TOPICS").split(",", -1)) { + SymbolRef extension = topic(name).symbol(); + if (!seen.add(extension)) { + throw xml.error(INVALID_WIRE, "Repeated XTF extension topic " + name); + } + extensions.add(extension); + } + } + var consistency = consistency(attributes); + basket = id; + state = State.IN_BASKET; + return new StartBasket(id, topic.symbol(), extensions, BasketKind.FULL, consistency, + Optional.empty(), Optional.empty(), stableId(id.value()), List.of()); + } + + private TransferEvent objectOrEndBasket() throws XMLStreamException { + int token = xml.nextTag(); + if (token == END_ELEMENT) { + var event = new EndBasket(basket); + basket = null; + state = State.BETWEEN_BASKETS; + return event; + } + String tag = elementName(token); + var descriptor = objects.get(tag); + if (descriptor == null || !descriptor.transferable()) { + throw xml.error(INVALID_WIRE, "Unknown or nontransferable XTF object tag " + tag); + } + if (descriptor.kind() != SymbolKind.CLASS) { + throw xml.error(UNSUPPORTED, "XTF association payloads are unsupported"); + } + var attributes = attributes(false, "TID", "BID", "OPERATION", "CONSISTENCY"); + if (attributes.containsKey("OPERATION")) { + throw xml.error(INVALID_WIRE, "A FULL object cannot carry an incremental OPERATION"); + } + String tid = identifier(required(attributes, "TID")); + var originalBasket = Optional.ofNullable(attributes.get("BID")).map(this::basketId); + var consistency = consistency(attributes); + var identity = new TransferIdentity(basket, originalBasket, Optional.of(new TransferId(tid, IdOrigin.SOURCE)), + stableId(tid).map(ObjectOid::new), source); + token = xml.nextTag(); + if (token == START_ELEMENT) { + throw xml.error(UNSUPPORTED, "XTF object field " + elementName(token) + " is unsupported"); + } + require(END_ELEMENT, token, tag); + return new ObjectEvent(new TransferObject(identity, + new TransferValue.Structured(descriptor.symbol(), Map.of())), Optional.empty(), consistency); + } + + private TopicDescriptor topic(String name) { + var topic = topics.get(name); + if (topic == null) { + throw xml.error(INVALID_WIRE, "Unknown XTF topic " + name); + } + return topic; + } + + private BasketId basketId(String value) { + return new BasketId(identifier(value), IdOrigin.SOURCE); + } + + private String identifier(String value) { + if (value.length() > MAX_IDENTIFIER) { + throw xml.error(LIMIT_EXCEEDED, "XTF identifier exceeds " + MAX_IDENTIFIER + " characters"); + } + if (!XML_ID.matcher(value).matches()) { + throw xml.error(INVALID_WIRE, "Invalid XTF identifier"); + } + int separator = value.indexOf(':'); + if (separator >= 0 && !oidSpaces.containsKey(value.substring(0, separator))) { + throw xml.error(INVALID_WIRE, "Unknown XTF OIDSPACE prefix"); + } + return value; + } + + private static Optional stableId(String value) { + return value.indexOf(':') >= 0 ? Optional.of(value) : Optional.empty(); + } + + private Consistency consistency(Map attributes) { + try { + return Consistency.valueOf(attributes.getOrDefault("CONSISTENCY", "COMPLETE")); + } catch (IllegalArgumentException e) { + throw xml.error(INVALID_WIRE, "Unknown XTF CONSISTENCY value"); + } + } + + private boolean is(int event, int token, String tag) { + var cursor = xml.cursor(); + return event == token && NAMESPACE.equals(cursor.getNamespaceURI()) && tag.equals(cursor.getLocalName()); + } + + private void require(int event, int token, String tag) { + if (!is(event, token, tag)) { + throw xml.error(INVALID_WIRE, "Expected " + (event == END_ELEMENT ? "end of " : "") + tag); + } + } + + private String elementName(int token) { + var cursor = xml.cursor(); + if (token != START_ELEMENT || !NAMESPACE.equals(cursor.getNamespaceURI())) { + throw xml.error(INVALID_WIRE, "Expected an element in the XTF 2.3 namespace"); + } + return cursor.getLocalName(); + } + + private Map attributes(boolean allowSchemaLocation, String... names) { + var allowed = List.of(names); + var values = new HashMap(); + var cursor = xml.cursor(); + for (int index = 0; index < cursor.getAttributeCount(); index++) { + var attribute = cursor.getAttributeName(index); + String name = attribute.getLocalPart(); + String value = cursor.getAttributeValue(index); + if (value.length() > Xtf23Input.MAX_TEXT) { + throw xml.error(LIMIT_EXCEEDED, "XTF attribute exceeds the text limit"); + } + if (allowSchemaLocation && XMLConstants.W3C_XML_SCHEMA_INSTANCE_NS_URI.equals(attribute.getNamespaceURI()) + && name.equals("schemaLocation")) { + continue; + } + if (!attribute.getNamespaceURI().isEmpty() || !allowed.contains(name)) { + throw xml.error(INVALID_WIRE, "Unexpected XTF attribute " + attribute); + } + if (value.indexOf('\r') >= 0 || value.indexOf('\n') >= 0 || value.indexOf('\t') >= 0) { + throw xml.error(INVALID_WIRE, "XTF attribute " + name + " must be single-line text"); + } + values.put(name, value); + } + return values; + } + + private String required(Map attributes, String name) { + var value = attributes.get(name); + if (value == null) { + throw xml.error(INVALID_WIRE, "Missing XTF attribute " + name); + } + return value; + } + + private enum State { + BEFORE_TRANSFER, BETWEEN_BASKETS, IN_BASKET, DONE + } +} diff --git a/interlis-xtf23/src/main/java/module-info.java b/interlis-xtf23/src/main/java/module-info.java new file mode 100644 index 00000000..00baa447 --- /dev/null +++ b/interlis-xtf23/src/main/java/module-info.java @@ -0,0 +1,8 @@ +/** Streaming XML adapter for INTERLIS 2.3 transfers. */ +module guru.interlis.xtf23 { + requires java.xml; + requires guru.interlis.model; + requires transitive guru.interlis.transfer; + + exports guru.interlis.xtf23; +} diff --git a/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23HeaderTest.java b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23HeaderTest.java new file mode 100644 index 00000000..17f9896f --- /dev/null +++ b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23HeaderTest.java @@ -0,0 +1,130 @@ +package guru.interlis.xtf23; + +import guru.interlis.source.SourceId; +import guru.interlis.transfer.TransferEvent; +import guru.interlis.transfer.TransferPlan; +import guru.interlis.transfer.TransferProtocolException; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.util.List; +import java.util.Optional; + +import static guru.interlis.xtf23.Xtf23Exception.Reason.*; +import static guru.interlis.xtf23.Xtf23TestData.*; +import static org.junit.jupiter.api.Assertions.*; + +final class Xtf23HeaderTest { + private static TransferPlan plan; + + @BeforeAll + static void compileModel() throws IOException { + plan = referencePlan(); + } + + @Test + void readsHeaderAndFirstObjectFromPinnedIoxFixture() throws IOException { + var source = new SourceId("memory:/identity.xtf"); + try (var input = Files.newInputStream(fixture("identity.xtf")); + var reader = new Xtf23Reader(plan, input, Optional.of(source))) { + var start = assertInstanceOf(TransferEvent.StartTransfer.class, reader.readNext(TOKEN).orElseThrow()); + assertEquals(Optional.of("iox-reference"), start.sender()); + assertEquals(Optional.of(source), start.source()); + assertEquals(1, start.models().size()); + var model = start.models().getFirst(); + assertEquals("Reference23", plan.model(model.model()).orElseThrow().name().declared().toString()); + assertEquals(Optional.of("2026-11-06"), model.modelVersion()); + assertEquals(Optional.of("https://example.invalid/iox-reference"), model.issuer()); + var basket = assertInstanceOf(TransferEvent.StartBasket.class, reader.readNext(TOKEN).orElseThrow()); + assertEquals("records-1", basket.id().value()); + var parent = assertInstanceOf(TransferEvent.ObjectEvent.class, reader.readNext(TOKEN).orElseThrow()); + assertEquals("parent-1", parent.object().identity().tid().orElseThrow().value()); + assertEquals(Optional.of(source), parent.object().identity().source()); + assertEquals(basket.id(), parent.object().identity().currentBasket()); + } + } + + @Test + void readsEmptyAndNonemptyBasketsWithNamespacedTagsAndHeaderMetadata() { + String xml = document(MODELS + """ + + + one & two + """, """ + + + + + """).replace("", "") + .replace("", ""), + valid.replace("Reference23", "Unknown"), + valid.replace("2026-11-06", "2026-10-06"), + valid.replace("https://example.invalid/iox-reference", "https://example.invalid/other"), + valid.replace("", MODELS.substring(MODELS.indexOf("")) + + ""), + valid.replace("", ""), valid.substring(PROLOG.length()))) { + assertFailure(xml, INVALID_WIRE); + } + } + + @Test + void rejectsUnsupportedHeaderAndIncrementalBasket() { + assertFailure(document(MODELS + "", ""), UNSUPPORTED); + assertFailure(document(MODELS, ""), UNSUPPORTED); + assertFailure(document(MODELS + "", ""), INVALID_WIRE); + } + + @Test + void rejectsUnknownTopicsAndInconsistentBasketContext() { + assertFailure(document(MODELS, ""), INVALID_WIRE); + assertFailure(document(MODELS, ""), INVALID_WIRE); + try (var reader = new Xtf23Reader(plan, input(document(MODELS, + "")))) { + var failure = assertThrows(TransferProtocolException.class, () -> readAll(reader)); + assertEquals(TransferProtocolException.Reason.CONTEXT_MISMATCH, failure.reason()); + } + } + + private static void assertFailure(String xml, Xtf23Exception.Reason expected) { + try (var reader = new Xtf23Reader(plan, input(xml))) { + var failure = assertThrows(Xtf23Exception.class, () -> readAll(reader)); + assertEquals(expected, failure.reason(), failure::getMessage); + } + } +} diff --git a/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23StreamLifecycleTest.java b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23StreamLifecycleTest.java new file mode 100644 index 00000000..ae6bc572 --- /dev/null +++ b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23StreamLifecycleTest.java @@ -0,0 +1,155 @@ +package guru.interlis.xtf23; + +import guru.interlis.execution.CancellationReason; +import guru.interlis.execution.CancellationToken; +import guru.interlis.execution.OperationCancelledException; +import guru.interlis.transfer.TransferEvent; +import guru.interlis.transfer.TransferPlan; +import guru.interlis.transfer.TransferProtocolException; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +import static guru.interlis.xtf23.Xtf23Exception.Reason.*; +import static guru.interlis.xtf23.Xtf23TestData.*; +import static org.junit.jupiter.api.Assertions.*; + +final class Xtf23StreamLifecycleTest { + private static TransferPlan plan; + + @BeforeAll + static void compileModel() throws IOException { + plan = referencePlan(); + } + + @Test + void confirmsCompleteXmlBeforeEndTransferAndLeavesInputOpen() { + var source = new ObservedInput(document(MODELS, "") + ""); + try (var reader = new Xtf23Reader(plan, source)) { + assertEquals(source.originalLength, source.available()); + assertInstanceOf(TransferEvent.StartTransfer.class, reader.readNext(TOKEN).orElseThrow()); + assertInstanceOf(TransferEvent.EndTransfer.class, reader.readNext(TOKEN).orElseThrow()); + assertTrue(reader.readNext(TOKEN).isEmpty()); + assertTrue(reader.readNext(TOKEN).isEmpty()); + assertEquals(0, source.available()); + } + assertEquals(0, source.closes); + } + + @Test + void rejectsPrematureEofMisnestingAndTrailingContent() { + String valid = document(MODELS, ""); + for (String broken : new String[] { + valid.replace("", ""), + valid.replace("", ""), + valid + "", + valid + "unexpected" + }) { + try (var reader = new Xtf23Reader(plan, input(broken))) { + assertInstanceOf(TransferEvent.StartTransfer.class, reader.readNext(TOKEN).orElseThrow()); + assertEquals(MALFORMED_XML, assertThrows(Xtf23Exception.class, + () -> reader.readNext(TOKEN)).reason()); + assertEquals(TransferProtocolException.Reason.FAILED, + assertThrows(TransferProtocolException.class, () -> reader.readNext(TOKEN)).reason()); + } + } + } + + @Test + void rejectsDoctypeAndDoesNotRequireResolvingSchemaHints() { + String valid = document(MODELS, ""); + for (String declaration : new String[] { + "", + "]>" + }) { + try (var reader = new Xtf23Reader(plan, input(valid.replace(PROLOG, PROLOG + declaration)))) { + assertEquals(UNSAFE_XML, assertThrows(Xtf23Exception.class, () -> reader.readNext(TOKEN)).reason()); + } + } + try (var reader = new Xtf23Reader(plan, input(valid.replace(" reader.readNext(cancelled)); + assertEquals(source.originalLength, source.available()); + assertInstanceOf(TransferEvent.StartTransfer.class, reader.readNext(TOKEN).orElseThrow()); + } finally { + reader.close(); + reader.close(); + } + assertEquals(0, source.closes); + assertEquals(TransferProtocolException.Reason.CLOSED, + assertThrows(TransferProtocolException.class, () -> reader.readNext(TOKEN)).reason()); + } + + @Test + void treatsCancellationDuringInputAsTerminal() { + var source = new ObservedInput(document(MODELS, "")); + try (var reader = new Xtf23Reader(plan, source)) { + assertThrows(OperationCancelledException.class, + () -> reader.readNext(() -> source.available() < source.originalLength)); + assertEquals(TransferProtocolException.Reason.FAILED, + assertThrows(TransferProtocolException.class, () -> reader.readNext(TOKEN)).reason()); + } + assertEquals(0, source.closes); + } + + @Test + void distinguishesIoFailuresAndTextLimitsFromXmlErrors() { + try (var reader = new Xtf23Reader(plan, new InputStream() { + @Override public int read() throws IOException { throw new IOException("source failed"); } + })) { + assertEquals(IO, assertThrows(Xtf23Exception.class, () -> reader.readNext(TOKEN)).reason()); + } + try (var reader = new Xtf23Reader(plan, input(document(MODELS + + "" + "x".repeat(Xtf23Input.MAX_TEXT + 1) + "", "")))) { + assertEquals(LIMIT_EXCEEDED, assertThrows(Xtf23Exception.class, () -> reader.readNext(TOKEN)).reason()); + } + try (var reader = new Xtf23Reader(plan, input(document(MODELS, "").replace(PROLOG, + PROLOG + "")))) { + assertEquals(LIMIT_EXCEEDED, assertThrows(Xtf23Exception.class, () -> reader.readNext(TOKEN)).reason()); + } + } + + @Test + void requiresUtf8AndRejectsInvalidUtf8AsMalformedXml() { + String valid = document(MODELS, ""); + var utf16 = valid.replace("UTF-8", "UTF-16").getBytes(StandardCharsets.UTF_16); + try (var reader = new Xtf23Reader(plan, new ByteArrayInputStream(utf16))) { + assertEquals(INVALID_WIRE, assertThrows(Xtf23Exception.class, () -> reader.readNext(TOKEN)).reason()); + } + byte[] invalid = valid.replace("test & sender", "~").getBytes(StandardCharsets.UTF_8); + invalid[valid.replace("test & sender", "~").indexOf('~')] = (byte) 0xff; + try (var reader = new Xtf23Reader(plan, new ByteArrayInputStream(invalid))) { + assertEquals(MALFORMED_XML, assertThrows(Xtf23Exception.class, () -> reader.readNext(TOKEN)).reason()); + } + } + + private static final class ObservedInput extends ByteArrayInputStream { + private final int originalLength; + private int closes; + + private ObservedInput(String xml) { + super(xml.getBytes(StandardCharsets.UTF_8)); + originalLength = available(); + } + + @Override + public void close() { + closes++; + } + } +} diff --git a/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23TestData.java b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23TestData.java new file mode 100644 index 00000000..7f22ffe7 --- /dev/null +++ b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23TestData.java @@ -0,0 +1,79 @@ +package guru.interlis.xtf23; + +import guru.interlis.compiler.CompilationOutcome; +import guru.interlis.compiler.CompilationRequest; +import guru.interlis.compiler.CompilerEngines; +import guru.interlis.compiler.CompilerOptions; +import guru.interlis.compiler.ModelSelector; +import guru.interlis.compiler.SourceSet; +import guru.interlis.execution.CancellationToken; +import guru.interlis.source.SourceId; +import guru.interlis.source.SourceRevision; +import guru.interlis.source.SourceText; +import guru.interlis.transfer.TransferEvent; +import guru.interlis.transfer.TransferPlan; +import guru.interlis.transfer.TransferPlanner; +import guru.interlis.transfer.TransferPlanningResult; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +final class Xtf23TestData { + static final CancellationToken TOKEN = CancellationToken.none(); + static final String PROLOG = ""; + static final String MODELS = """ + + """; + + private Xtf23TestData() { } + + static Path fixture(String name) { + return Path.of(System.getProperty("testdata.dir"), "reference", "iox", "ili23", name); + } + + static TransferPlan referencePlan() throws IOException { + Path path = fixture("Reference23.ili"); + return compile("Reference23", new SourceText(new SourceId(path.toUri().toString()), + new SourceRevision(1), Files.readString(path))); + } + + private static TransferPlan compile(String model, SourceText source) { + var request = new CompilationRequest(List.of(new ModelSelector.ByName(model, Optional.empty(), Optional.empty())), + SourceSet.of(Map.of(source.id(), source)), CompilerOptions.defaults()); + var outcome = CompilerEngines.create().compile(request, TOKEN); + var schema = assertInstanceOf(CompilationOutcome.Succeeded.class, outcome, + () -> outcome instanceof CompilationOutcome.Rejected rejected + ? rejected.analysis().requestDiagnostics().toString() : outcome.toString()).schema(); + return assertInstanceOf(TransferPlanningResult.Ready.class, TransferPlanner.plan(schema)).plan(); + } + + static String document(String headerContent, String baskets) { + return PROLOG + """ + + %s + %s + + """.formatted(headerContent, baskets); + } + + static ByteArrayInputStream input(String xml) { + return new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)); + } + + static List readAll(Xtf23Reader reader) { + var events = new ArrayList(); + for (var event = reader.readNext(TOKEN); event.isPresent(); event = reader.readNext(TOKEN)) { + events.add(event.orElseThrow()); + } + return List.copyOf(events); + } +} diff --git a/pom.xml b/pom.xml index a3118c0f..151a81e2 100644 --- a/pom.xml +++ b/pom.xml @@ -12,6 +12,7 @@ interlis-projection interlis-documentation interlis-transfer + interlis-xtf23 interlis-wasm-bridge interlis-compiler-cli interlis-lsp From da2d5920462dff274954001ccfa1399a9a7097ac Mon Sep 17 00:00:00 2001 From: Stefan Ziegler Date: Fri, 6 Nov 2026 22:05:51 +0100 Subject: [PATCH 2/6] Decode XTF 2.3 objects, references and associations --- .../build.jenesis/test-inputs.properties | 1 + .../guru/interlis/xtf23/Xtf23Exception.java | 2 +- .../interlis/xtf23/Xtf23IdentityScope.java | 24 ++ .../java/guru/interlis/xtf23/Xtf23Layout.java | 174 +++++++++++ .../guru/interlis/xtf23/Xtf23Lexemes.java | 40 +++ .../java/guru/interlis/xtf23/Xtf23Reader.java | 286 +++++++++++++++++- .../xtf23/Xtf23IdentityScopeTest.java | 69 +++++ .../xtf23/Xtf23ObjectEncodingTest.java | 234 ++++++++++++++ .../guru/interlis/xtf23/Xtf23TestData.java | 4 + 9 files changed, 819 insertions(+), 15 deletions(-) create mode 100644 interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23IdentityScope.java create mode 100644 interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Layout.java create mode 100644 interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Lexemes.java create mode 100644 interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23IdentityScopeTest.java create mode 100644 interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23ObjectEncodingTest.java diff --git a/interlis-xtf23/build.jenesis/test-inputs.properties b/interlis-xtf23/build.jenesis/test-inputs.properties index 120c2530..921b279b 100644 --- a/interlis-xtf23/build.jenesis/test-inputs.properties +++ b/interlis-xtf23/build.jenesis/test-inputs.properties @@ -1,2 +1,3 @@ input.0=testdata/reference/iox/ili23/Reference23.ili input.1=testdata/reference/iox/ili23/identity.xtf +input.2=testdata/reference/iox/ili23/unique-across-baskets.xtf diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Exception.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Exception.java index 85dc813a..e378ea64 100644 --- a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Exception.java +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Exception.java @@ -4,7 +4,7 @@ public final class Xtf23Exception extends IllegalStateException { /** Distinguishes format failures from an unsupported feature or host failure. */ public enum Reason { - MALFORMED_XML, UNSAFE_XML, INVALID_WIRE, UNSUPPORTED, LIMIT_EXCEEDED, IO, CONFIGURATION + MALFORMED_XML, UNSAFE_XML, INVALID_WIRE, DUPLICATE_ID, UNSUPPORTED, LIMIT_EXCEEDED, IO, CONFIGURATION } private final Reason reason; diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23IdentityScope.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23IdentityScope.java new file mode 100644 index 00000000..fd2937a3 --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23IdentityScope.java @@ -0,0 +1,24 @@ +package guru.interlis.xtf23; + +import guru.interlis.transfer.TransferIdentity.BasketId; +import guru.interlis.transfer.TransferIdentity.TransferId; + +import java.util.HashSet; +import java.util.Set; + +/** One FULL transfer's shared BID/TID space. The wire spelling is independent of IdOrigin. */ +final class Xtf23IdentityScope { + private Set ids = new HashSet<>(); + + boolean claim(BasketId id) { + return ids.add(id.value()); + } + + boolean claim(TransferId id) { + return ids.add(id.value()); + } + + void release() { + ids = Set.of(); + } +} diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Layout.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Layout.java new file mode 100644 index 00000000..f54adb4f --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Layout.java @@ -0,0 +1,174 @@ +package guru.interlis.xtf23; + +import guru.interlis.model.CardinalityUpper; +import guru.interlis.model.Modifier; +import guru.interlis.model.SymbolId; +import guru.interlis.model.SymbolKind; +import guru.interlis.model.SymbolRef; +import guru.interlis.transfer.AttributeSlot; +import guru.interlis.transfer.ObjectDescriptor; +import guru.interlis.transfer.RoleSlot; +import guru.interlis.transfer.TransferPlan; +import guru.interlis.transfer.TransferSlot; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** Derives the XTF 2.3 wire order and association placement from the neutral plan. */ +final class Xtf23Layout { + record Embedded(ObjectDescriptor association, RoleSlot owner, RoleSlot opposite, RoleSlot rootOpposite) { } + + record Field(TransferSlot slot, List embeddings, int level) { + String name() { + return embeddings.isEmpty() ? slot.name().declared().last() : embeddings.getFirst().opposite().name().declared().last(); + } + + private int group() { + return embeddings.isEmpty() ? (slot instanceof RoleSlot ? 0 : 1) : 2; + } + } + + private static final Comparator WIRE_ORDER = Comparator.comparingInt(Field::level) + .thenComparingInt(Field::group) + .thenComparing(field -> field.embeddings().isEmpty() ? "" : field.name()) + .thenComparingInt(field -> field.slot().member().position()); + + private final TransferPlan plan; + private final Map embedded; + + Xtf23Layout(TransferPlan plan) { + this.plan = plan; + var result = new HashMap(); + for (var descriptor : plan.objects()) { + if (descriptor.kind() == SymbolKind.ASSOCIATION && descriptor.transferable()) { + placement(descriptor).ifPresent(value -> result.put(descriptor.symbol(), value)); + } + } + embedded = Map.copyOf(result); + } + + Optional embedded(ObjectDescriptor descriptor) { + return Optional.ofNullable(embedded.get(descriptor.symbol())); + } + + boolean identified(ObjectDescriptor association) { + if (!association.oidRules().isEmpty()) { + return association.oidRules().stream().anyMatch(rule -> !rule.basket() && rule.domain().isPresent()); + } + return ancestors(association).stream().anyMatch(descriptor -> descriptor.modifiers().contains(Modifier.OID)); + } + + List fields(ObjectDescriptor descriptor, Set activeTypes, boolean payloadOnly) { + var inheritance = ancestors(descriptor); + var levels = new HashMap(); + for (int i = 0; i < inheritance.size(); i++) { + for (var slot : inheritance.get(i).slots()) { + levels.putIfAbsent(slot.member().position(), i); + } + } + var fields = new ArrayList(); + for (var slot : descriptor.slots()) { + if (slot instanceof AttributeSlot attribute && !attribute.attribute().transientAttribute() + || slot instanceof RoleSlot && !payloadOnly) { + fields.add(new Field(slot, List.of(), levels.get(slot.member().position()))); + } + } + if (!payloadOnly) { + addEmbedded(fields, descriptor, activeTypes, inheritance); + } + fields.sort(WIRE_ORDER); + return List.copyOf(fields); + } + + private void addEmbedded(List fields, ObjectDescriptor descriptor, Set activeTypes, + List inheritance) { + if (descriptor.kind() == SymbolKind.STRUCTURE) { + return; + } + var byName = new HashMap>(); + for (var entry : embedded.entrySet()) { + var value = entry.getValue(); + if (activeTypes.contains(entry.getKey()) && matchesRoots(descriptor, value.owner().role().targetRoots())) { + byName.computeIfAbsent(value.opposite().name().declared().last(), unused -> new ArrayList<>()).add(value); + } + } + for (var variants : byName.values()) { + // Resolve concrete variants in a stable order even if the plan was indexed in a hash map. + variants.sort(Comparator.comparing(value -> value.association().name().declared().toString())); + var rootRole = variants.getFirst().rootOpposite(); + int level = inheritance.size() - 1; + for (var variant : variants) { + var targets = variant.owner().role().targets().stream().map(target -> target.base()).toList(); + for (int i = 0; i < level; i++) { + if (targets.contains(inheritance.get(i).symbol().symbol())) { + level = i; + break; + } + } + } + fields.add(new Field(rootRole, List.copyOf(variants), level)); + } + } + + boolean matchesRoots(ObjectDescriptor descriptor, List allowed) { + return allowed.isEmpty() || ancestors(descriptor).stream() + .anyMatch(base -> allowed.contains(base.symbol().symbol())); + } + + private Optional placement(ObjectDescriptor association) { + if (identified(association)) { + return Optional.empty(); + } + var root = ancestors(association).getFirst(); + var rootRoles = roles(root); + var effective = roles(association); + if (rootRoles.size() != 2 || effective.size() != 2) { + return Optional.empty(); + } + boolean firstMany = many(rootRoles.get(0)); + boolean secondMany = many(rootRoles.get(1)); + if (firstMany && secondMany) { + return Optional.empty(); + } + int owner = firstMany ? 0 : 1; + if (!local(root, rootRoles.get(owner))) { + if (firstMany || secondMany || !local(root, rootRoles.getFirst())) { + return Optional.empty(); + } + owner = 0; + } + int opposite = 1 - owner; + return Optional.of(new Embedded(association, effective.get(owner), effective.get(opposite), rootRoles.get(opposite))); + } + + private boolean local(ObjectDescriptor association, RoleSlot role) { + return role.role().targets().stream().allMatch(target -> + plan.object(new SymbolRef(plan.snapshotId(), target.base())) + .filter(object -> object.topic().equals(association.topic())).isPresent()); + } + + private static boolean many(RoleSlot role) { + return !(role.role().cardinality().upper() instanceof CardinalityUpper.Finite finite) + || finite.value().compareTo(BigInteger.ONE) > 0; + } + + private static List roles(ObjectDescriptor association) { + return association.slots().stream().filter(RoleSlot.class::isInstance).map(RoleSlot.class::cast).toList(); + } + + private List ancestors(ObjectDescriptor descriptor) { + var hierarchy = new ArrayList(); + hierarchy.add(descriptor); + while (descriptor.baseType().isPresent()) { + descriptor = plan.object(descriptor.baseType().orElseThrow()).orElseThrow(); + hierarchy.add(descriptor); + } + return hierarchy.reversed(); + } +} diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Lexemes.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Lexemes.java new file mode 100644 index 00000000..34312577 --- /dev/null +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Lexemes.java @@ -0,0 +1,40 @@ +package guru.interlis.xtf23; + +import guru.interlis.transfer.TransferValue.NumberValue; + +import java.math.BigDecimal; +import java.util.Arrays; +import java.util.regex.Pattern; + +/** Lexical rules for XTF values, independent of model constraints. */ +final class Xtf23Lexemes { + private static final Pattern DECIMAL = + Pattern.compile("[+-]?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?(?:[eE][+-]?[0-9]+)?"); + private static final Pattern NAME = Pattern.compile("[A-Za-z][A-Za-z0-9_]*"); + private static final Pattern STRUCTURED_NUMBER = Pattern.compile("[+-]?[0-9]++(?::[0-9]++)++(?:\\.[0-9]++)?"); + static final int MAX_NUMERIC_LENGTH = 4096; + static final Pattern POSITION = Pattern.compile("[0-9]+"); + + private Xtf23Lexemes() { } + + static boolean singleLine(String value) { + return value.indexOf('\r') < 0 && value.indexOf('\n') < 0 && value.indexOf('\t') < 0; + } + + static boolean enumeration(String value) { + return value.length() <= 4096 && !value.equals("OTHERS") + && Arrays.stream(value.split("\\.", -1)).allMatch(part -> NAME.matcher(part).matches()); + } + + static NumberValue number(String value) { + if (!DECIMAL.matcher(value).matches()) { + throw new NumberFormatException("Invalid XTF decimal"); + } + return new NumberValue(value, new BigDecimal(value)); + } + + static boolean nonDecimalNumber(String value) { + return value.equals("PI") || value.equals("LNBASE") + || (value.indexOf(':') >= 0 && STRUCTURED_NUMBER.matcher(value).matches()); + } +} diff --git a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Reader.java b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Reader.java index e022a127..d869a0ae 100644 --- a/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Reader.java +++ b/interlis-xtf23/src/main/java/guru/interlis/xtf23/Xtf23Reader.java @@ -1,13 +1,25 @@ package guru.interlis.xtf23; import guru.interlis.execution.CancellationToken; +import guru.interlis.model.CollectionType; +import guru.interlis.model.DomainReferenceType; +import guru.interlis.model.DomainSymbol; import guru.interlis.model.LanguageVersion; import guru.interlis.model.ModelSymbol; +import guru.interlis.model.Modifier; +import guru.interlis.model.ReferenceType; +import guru.interlis.model.StructureValueType; import guru.interlis.model.SymbolKind; import guru.interlis.model.SymbolRef; +import guru.interlis.model.TextKind; +import guru.interlis.model.TextType; +import guru.interlis.model.TypeDescriptor; +import guru.interlis.model.TypeRef; import guru.interlis.source.SourceId; +import guru.interlis.transfer.AttributeSlot; import guru.interlis.transfer.ModelDescriptor; import guru.interlis.transfer.ObjectDescriptor; +import guru.interlis.transfer.RoleSlot; import guru.interlis.transfer.TopicDescriptor; import guru.interlis.transfer.TransferEvent; import guru.interlis.transfer.TransferIdentity; @@ -20,6 +32,7 @@ import javax.xml.XMLConstants; import javax.xml.stream.XMLStreamException; import java.io.InputStream; +import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -28,6 +41,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -43,6 +57,8 @@ * Events are immutable and can be retained after the reader advances. * The XML reader bounds depth to 128 elements, scalar text to 1,048,576 UTF-16 * code units, and input read during any individual parser advance to 8 MiB. + * The reader holds a transfer-wide index of occurrence IDs whose size grows with + * the number of distinct BIDs and TIDs. */ public final class Xtf23Reader extends TransferReader { static final String NAMESPACE = "http://www.interlis.ch/INTERLIS2.3"; @@ -57,7 +73,11 @@ public final class Xtf23Reader extends TransferReader { private final Map models; private final Map topics; private final Map objects; + private final Xtf23Layout layout; + private final Xtf23IdentityScope identities = new Xtf23IdentityScope(); + private final Map> fieldIndexes = new HashMap<>(); private Map oidSpaces = Map.of(); + private Set activeTypes = Set.of(); private List oidSpaceDeclarations = List.of(); private State state = State.BEFORE_TRANSFER; private BasketId basket; @@ -76,6 +96,7 @@ public Xtf23Reader(TransferPlan plan, InputStream input, Optional sour models = index(plan.models(), ModelDescriptor::name); topics = index(plan.topics(), TopicDescriptor::name); objects = index(plan.objects(), ObjectDescriptor::name); + layout = new Xtf23Layout(plan); } /** Returns the immutable, ordered OIDSPACE declarations after reading the header. @@ -100,7 +121,11 @@ protected Optional readEvent(CancellationToken cancellation) { case DONE -> Optional.empty(); }; } catch (XMLStreamException failure) { + clearTransferState(); throw xml.translate(failure); + } catch (RuntimeException failure) { + clearTransferState(); + throw failure; } finally { xml.finish(); } @@ -108,9 +133,20 @@ protected Optional readEvent(CancellationToken cancellation) { @Override protected void release() { + clearTransferState(); + xml.close(); + } + + private void clearTransferState() { basket = null; oidSpaces = Map.of(); - xml.close(); + identities.release(); + clearBasketState(); + } + + private void clearBasketState() { + activeTypes = Set.of(); + fieldIndexes.clear(); } private StartTransfer startTransfer() throws XMLStreamException { @@ -228,7 +264,7 @@ private TransferEvent basketOrEndTransfer() throws XMLStreamException { throw xml.error(INVALID_WIRE, "Unexpected XML after the XTF transfer"); } state = State.DONE; - oidSpaces = Map.of(); + clearTransferState(); return new EndTransfer(); } String tag = elementName(token); @@ -255,6 +291,14 @@ private TransferEvent basketOrEndTransfer() throws XMLStreamException { } } var consistency = consistency(attributes); + if (!identities.claim(id)) { + throw xml.error(DUPLICATE_ID, "XTF FULL BID/TID collision for " + id.value()); + } + var allowed = new HashSet<>(topic.objectTypes()); + for (var extension : extensions) { + allowed.addAll(plan.topic(extension).orElseThrow().objectTypes()); + } + activeTypes = Set.copyOf(allowed); basket = id; state = State.IN_BASKET; return new StartBasket(id, topic.symbol(), extensions, BasketKind.FULL, consistency, @@ -266,6 +310,7 @@ private TransferEvent objectOrEndBasket() throws XMLStreamException { if (token == END_ELEMENT) { var event = new EndBasket(basket); basket = null; + clearBasketState(); state = State.BETWEEN_BASKETS; return event; } @@ -274,25 +319,238 @@ private TransferEvent objectOrEndBasket() throws XMLStreamException { if (descriptor == null || !descriptor.transferable()) { throw xml.error(INVALID_WIRE, "Unknown or nontransferable XTF object tag " + tag); } - if (descriptor.kind() != SymbolKind.CLASS) { - throw xml.error(UNSUPPORTED, "XTF association payloads are unsupported"); + if (layout.embedded(descriptor).isPresent()) { + throw xml.error(INVALID_WIRE, "Embedded XTF association cannot occur as a standalone object: " + tag); } var attributes = attributes(false, "TID", "BID", "OPERATION", "CONSISTENCY"); if (attributes.containsKey("OPERATION")) { throw xml.error(INVALID_WIRE, "A FULL object cannot carry an incremental OPERATION"); } - String tid = identifier(required(attributes, "TID")); + boolean requiresTid = descriptor.kind() == SymbolKind.CLASS || layout.identified(descriptor); + if (requiresTid != attributes.containsKey("TID")) { + throw xml.error(INVALID_WIRE, requiresTid ? "Missing XTF attribute TID" + : "A link without its own identification cannot carry TID"); + } + var tid = Optional.ofNullable(attributes.get("TID")) + .map(value -> new TransferId(identifier(value), IdOrigin.SOURCE)); + tid.ifPresent(value -> { + if (!identities.claim(value)) { + throw xml.error(DUPLICATE_ID, "XTF FULL BID/TID collision for " + value.value()); + } + }); var originalBasket = Optional.ofNullable(attributes.get("BID")).map(this::basketId); var consistency = consistency(attributes); - var identity = new TransferIdentity(basket, originalBasket, Optional.of(new TransferId(tid, IdOrigin.SOURCE)), - stableId(tid).map(ObjectOid::new), source); - token = xml.nextTag(); - if (token == START_ELEMENT) { - throw xml.error(UNSUPPORTED, "XTF object field " + elementName(token) + " is unsupported"); - } - require(END_ELEMENT, token, tag); + var identity = new TransferIdentity(basket, originalBasket, tid, + tid.flatMap(value -> stableId(value.value())).map(ObjectOid::new), source); + var data = readFields(descriptor, tag, false); return new ObjectEvent(new TransferObject(identity, - new TransferValue.Structured(descriptor.symbol(), Map.of())), Optional.empty(), consistency); + new TransferValue.Structured(descriptor.symbol(), data)), Optional.empty(), consistency); + } + + private record PositionedField(Xtf23Layout.Field field, int position) { } + + private static Map indexFields(List fields) { + var index = new HashMap(); + for (int i = 0; i < fields.size(); i++) { + var field = fields.get(i); + if (index.putIfAbsent(field.name(), new PositionedField(field, i)) != null) { + index.put(field.name(), new PositionedField(field, -1)); + } + } + return index; + } + + private Map readFields(ObjectDescriptor descriptor, String tag, boolean payloadOnly) + throws XMLStreamException { + var fields = payloadOnly ? indexFields(layout.fields(descriptor, activeTypes, true)) + : fieldIndexes.computeIfAbsent(descriptor.symbol(), unused -> indexFields(layout.fields(descriptor, activeTypes, false))); + var data = new LinkedHashMap(); + int lastPosition = -1; + int token = xml.nextTag(); + while (!is(END_ELEMENT, token, tag)) { + String name = elementName(token); + var positioned = fields.get(name); + if (positioned == null) { + throw xml.error(INVALID_WIRE, "Unknown XTF field " + name + " on " + tag); + } + if (positioned.position() < 0) { + throw xml.error(UNSUPPORTED, "Ambiguous XTF field " + name + " on " + tag); + } + if (positioned.position() <= lastPosition) { + throw xml.error(INVALID_WIRE, "Repeated or out-of-order XTF field " + name); + } + lastPosition = positioned.position(); + var field = positioned.field(); + var value = field.embeddings().isEmpty() ? switch (field.slot()) { + case AttributeSlot attribute -> readValue(type(attribute.effectiveType()), name); + case RoleSlot role -> readRole(role, name); + } : readEmbedded(field, name); + data.put(field.slot().symbol(), value); + token = xml.nextTag(); + } + return data; + } + + private TypeDescriptor type(TypeRef ref) { + var descriptor = plan.type(ref).known().orElseThrow(); + while (descriptor instanceof DomainReferenceType alias) { + var domain = (DomainSymbol) plan.symbol(new SymbolRef(plan.snapshotId(), alias.targetDomain())) + .known().orElseThrow(); + descriptor = plan.type(new TypeRef(plan.snapshotId(), domain.typeId())).known().orElseThrow(); + } + return descriptor; + } + + private TransferValue readValue(TypeDescriptor type, String tag) throws XMLStreamException { + return switch (type.kind()) { + case TEXT -> readText(((TextType) type).textKind() == TextKind.MTEXT); + case FORMATTED -> readText(false); + case BOOLEAN -> readBoolean(); + case ENUMERATION, ALIGNMENT -> readEnumeration(); + case NUMERIC -> { + attributes(false); + yield readNumber(); + } + case REFERENCE -> { + var result = reference(attributes(false, "REF", "BID"), ((ReferenceType) type).external(), false); + require(END_ELEMENT, xml.nextTag(), tag); + yield result; + } + case STRUCTURE_VALUE -> { + attributes(false); + var result = readStructure((StructureValueType) type, xml.nextTag()); + require(END_ELEMENT, xml.nextTag(), tag); + yield result; + } + case COLLECTION -> readCollection((CollectionType) type, tag); + default -> throw xml.error(UNSUPPORTED, "Unsupported XTF type " + type.kind() + " for " + tag); + }; + } + + private TransferValue.Text readText(boolean multiline) throws XMLStreamException { + attributes(false); + String value = xml.text(); + if (!multiline && !Xtf23Lexemes.singleLine(value)) { + throw xml.error(INVALID_WIRE, "Expected single-line XTF text"); + } + return new TransferValue.Text(value); + } + + private TransferValue.BooleanValue readBoolean() throws XMLStreamException { + attributes(false); + return switch (xml.text()) { + case "true" -> new TransferValue.BooleanValue(true); + case "false" -> new TransferValue.BooleanValue(false); + default -> throw xml.error(INVALID_WIRE, "Expected XTF BOOLEAN true or false"); + }; + } + + private TransferValue.Enumeration readEnumeration() throws XMLStreamException { + attributes(false); + String value = xml.text(); + if (!Xtf23Lexemes.enumeration(value)) { + throw xml.error(INVALID_WIRE, "Invalid XTF enumeration path"); + } + return new TransferValue.Enumeration(List.of(value.split("\\."))); + } + + private TransferValue.NumberValue readNumber() throws XMLStreamException { + String value = xml.text(); + if (value.length() > Xtf23Lexemes.MAX_NUMERIC_LENGTH) { + throw xml.error(LIMIT_EXCEEDED, "XTF numeric lexeme exceeds the numeric length limit"); + } + if (Xtf23Lexemes.nonDecimalNumber(value)) { + throw xml.error(UNSUPPORTED, "Nondecimal XTF numbers are unsupported"); + } + try { + return Xtf23Lexemes.number(value); + } catch (NumberFormatException e) { + throw xml.error(INVALID_WIRE, "Invalid XTF decimal value"); + } + } + + private TransferValue.Structured readStructure(StructureValueType expected, int token) throws XMLStreamException { + String tag = elementName(token); + var descriptor = objects.get(tag); + if (descriptor == null || descriptor.kind() != SymbolKind.STRUCTURE + || descriptor.modifiers().contains(Modifier.ABSTRACT) || !layout.matchesRoots(descriptor, expected.targetRoots())) { + throw xml.error(INVALID_WIRE, "Unexpected concrete XTF structure " + tag); + } + attributes(false); + return new TransferValue.Structured(descriptor.symbol(), readFields(descriptor, tag, false)); + } + + private TransferValue.Repeated readCollection(CollectionType collection, String tag) throws XMLStreamException { + attributes(false); + if (!(type(new TypeRef(plan.snapshotId(), collection.elementType())) instanceof StructureValueType structure)) { + throw xml.error(UNSUPPORTED, "XTF 2.3 collections require structure values"); + } + var values = new ArrayList(); + int token = xml.nextTag(); + while (!is(END_ELEMENT, token, tag)) { + values.add(readStructure(structure, token)); + token = xml.nextTag(); + } + if (values.isEmpty()) { + throw xml.error(INVALID_WIRE, "Present XTF collection requires at least one structure"); + } + return new TransferValue.Repeated(collection.collectionKind(), values); + } + + private TransferValue.Reference readRole(RoleSlot role, String tag) throws XMLStreamException { + var value = reference(attributes(false, "REF", "BID", "ORDER_POS"), role.role().external(), role.role().ordered()); + require(END_ELEMENT, xml.nextTag(), tag); + return value; + } + + private TransferValue readEmbedded(Xtf23Layout.Field field, String tag) throws XMLStreamException { + var attributes = attributes(false, "REF", "BID", "ORDER_POS"); + var variants = field.embeddings(); + var first = variants.getFirst(); + if (variants.stream().anyMatch(value -> !value.rootOpposite().symbol().equals(first.rootOpposite().symbol()))) { + throw xml.error(UNSUPPORTED, "Ambiguous embedded XTF role " + tag); + } + int token = xml.nextTag(); + if (is(END_ELEMENT, token, tag)) { + var role = first.opposite().role(); + if (variants.stream().anyMatch(value -> value.opposite().role().external() != role.external() + || value.opposite().role().ordered() != role.ordered())) { + throw xml.error(UNSUPPORTED, "Embedded XTF role needs a concrete association tag: " + tag); + } + return reference(attributes, role.external(), role.ordered()); + } + String associationTag = elementName(token); + var variant = variants.stream().filter(value -> value.association().name().declared().toString().equals(associationTag)) + .findFirst().orElseThrow(() -> xml.error(INVALID_WIRE, "Unexpected embedded XTF association " + associationTag)); + attributes(false); + var role = variant.opposite(); + var reference = reference(attributes, role.role().external(), role.role().ordered()); + var fields = readFields(variant.association(), associationTag, true); + require(END_ELEMENT, xml.nextTag(), tag); + var values = new LinkedHashMap(); + values.put(role.symbol(), reference); + values.putAll(fields); + return new TransferValue.Structured(variant.association().symbol(), values); + } + + private TransferValue.Reference reference(Map attributes, boolean external, boolean ordered) { + String ref = identifier(required(attributes, "REF")); + ObjectKey key = stableId(ref).map(ObjectOid::new).orElseGet(() -> new TransferId(ref, IdOrigin.SOURCE)); + var targetBasket = Optional.ofNullable(attributes.get("BID")).map(this::basketId); + var position = Optional.ofNullable(attributes.get("ORDER_POS")).map(value -> { + if (value.length() > Xtf23Lexemes.MAX_NUMERIC_LENGTH) { + throw xml.error(LIMIT_EXCEEDED, "XTF ORDER_POS exceeds the numeric length limit"); + } + if (!ordered || !Xtf23Lexemes.POSITION.matcher(value).matches()) { + throw xml.error(INVALID_WIRE, "ORDER_POS requires an ordered role and a positive integer"); + } + var parsed = new BigInteger(value); + if (parsed.signum() <= 0) { + throw xml.error(INVALID_WIRE, "ORDER_POS requires a positive integer"); + } + return parsed; + }); + return new TransferValue.Reference(key, targetBasket, external, position); } private TopicDescriptor topic(String name) { @@ -370,7 +628,7 @@ private Map attributes(boolean allowSchemaLocation, String... na if (!attribute.getNamespaceURI().isEmpty() || !allowed.contains(name)) { throw xml.error(INVALID_WIRE, "Unexpected XTF attribute " + attribute); } - if (value.indexOf('\r') >= 0 || value.indexOf('\n') >= 0 || value.indexOf('\t') >= 0) { + if (!Xtf23Lexemes.singleLine(value)) { throw xml.error(INVALID_WIRE, "XTF attribute " + name + " must be single-line text"); } values.put(name, value); diff --git a/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23IdentityScopeTest.java b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23IdentityScopeTest.java new file mode 100644 index 00000000..c1ee52a1 --- /dev/null +++ b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23IdentityScopeTest.java @@ -0,0 +1,69 @@ +package guru.interlis.xtf23; + +import guru.interlis.transfer.TransferEvent; +import guru.interlis.transfer.TransferPlan; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; + +import static guru.interlis.transfer.TransferIdentity.*; +import static guru.interlis.xtf23.Xtf23Exception.Reason.DUPLICATE_ID; +import static guru.interlis.xtf23.Xtf23TestData.*; +import static org.junit.jupiter.api.Assertions.*; + +final class Xtf23IdentityScopeTest { + private static TransferPlan plan; + + @BeforeAll + static void compileModel() throws IOException { + plan = referencePlan(); + } + + @Test + void detectsBothCollisionOrdersAcrossBaskets() { + for (String baskets : List.of( + """ + + + """, + """ + + + """, + """ + + """)) { + try (var reader = new Xtf23Reader(plan, input(document(MODELS, baskets)))) { + assertEquals(DUPLICATE_ID, assertThrows(Xtf23Exception.class, () -> readAll(reader)).reason()); + } + } + } + + @Test + void preservesDistinctLifetimesAndChecksTheSpellingIndependentlyOfOrigin() { + for (var basketOrigin : IdOrigin.values()) { + for (var tidOrigin : IdOrigin.values()) { + var basket = new BasketId("same", basketOrigin); + var object = new TransferId("same", tidOrigin); + var bidFirst = new Xtf23IdentityScope(); + assertTrue(bidFirst.claim(basket)); + assertFalse(bidFirst.claim(object)); + var tidFirst = new Xtf23IdentityScope(); + assertTrue(tidFirst.claim(object)); + assertFalse(tidFirst.claim(basket)); + } + } + String document = document(MODELS, """ + + + + """); + for (int i = 0; i < 2; i++) { + try (var reader = new Xtf23Reader(plan, input(document))) { + assertInstanceOf(TransferEvent.EndTransfer.class, readAll(reader).getLast()); + } + } + } +} diff --git a/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23ObjectEncodingTest.java b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23ObjectEncodingTest.java new file mode 100644 index 00000000..c6f7c8b0 --- /dev/null +++ b/interlis-xtf23/src/test/java/guru/interlis/xtf23/Xtf23ObjectEncodingTest.java @@ -0,0 +1,234 @@ +package guru.interlis.xtf23; + +import guru.interlis.model.CollectionKind; +import guru.interlis.model.SymbolRef; +import guru.interlis.transfer.TransferEvent.ObjectEvent; +import guru.interlis.transfer.TransferPlan; +import guru.interlis.transfer.TransferValue; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.file.Files; +import java.util.List; +import java.util.Optional; + +import static guru.interlis.transfer.TransferIdentity.*; +import static guru.interlis.transfer.TransferValue.*; +import static guru.interlis.xtf23.Xtf23Exception.Reason.*; +import static guru.interlis.xtf23.Xtf23TestData.*; +import static org.junit.jupiter.api.Assertions.*; + +final class Xtf23ObjectEncodingTest { + private static TransferPlan reference; + private static TransferPlan values; + private static final String VALUES = """ + + """; + private static final String OIDSPACES = """ + + """; + + @BeforeAll + static void compileModels() throws IOException { + reference = referencePlan(); + values = compileInline("Values", """ + INTERLIS 2.3; + MODEL Values (en) AT "https://example.invalid/values" VERSION "1" = + TOPIC Data = + CLASS Parent = END Parent; + STRUCTURE Detail = + label: TEXT*32; + owner: REFERENCE TO (EXTERNAL) Parent; + END Detail; + CLASS Item = + quantity: -100.000 .. 999.000; + flag: BOOLEAN; + shade: (dark (red, blue), light); + label: TEXT*32; + notes: MTEXT*64; + one: Detail; + details: BAG {0..*} OF Detail; + sequence: LIST {0..*} OF Detail; + END Item; + ASSOCIATION Note = + parent (EXTERNAL) -- {0..1} Parent; + item -- {0..*} Item; + remark: TEXT*32; + END Note; + ASSOCIATION Peer = + left (ORDERED, EXTERNAL) -- {0..*} Parent; + right -- {0..*} Item; + remark: TEXT*32; + END Peer; + ASSOCIATION Stable (OID) = + first -- {0..1} Parent; + second -- {0..*} Item; + END Stable; + END Data; + END Values. + """); + } + + @Test + void readsPinnedObjectsAndDoesNotEvaluateUniqueAcrossBaskets() throws IOException { + for (String fixture : List.of("identity.xtf", "unique-across-baskets.xtf")) { + try (var source = Files.newInputStream(fixture(fixture)); + var reader = new Xtf23Reader(reference, source)) { + var objects = objects(reader); + assertEquals(3, objects.size()); + var item = objects.get(1); + assertEquals(new Text("same"), field(reference, item, "code")); + var details = assertInstanceOf(Repeated.class, field(reference, item, "details")); + assertEquals(CollectionKind.LIST, details.kind()); + assertEquals(List.of(new Text("zéro & \\"), new Text("alpha")), + details.values().stream().map(value -> field(reference, (Structured) value, "description")).toList()); + var parent = assertInstanceOf(Reference.class, + item.value(slot(reference, "Reference23.Records.ParentLink", "parent")).orElseThrow()); + assertEquals(new TransferId("parent-1", IdOrigin.SOURCE), parent.target()); + assertEquals(Optional.of(new BasketId("records-1", IdOrigin.SOURCE)), parent.targetBasket()); + assertFalse(parent.external()); + assertEquals(new Text(fixture.equals("identity.xtf") ? "other" : "same"), + field(reference, objects.getLast(), "code")); + } + } + } + + @Test + void keepsScalarLexemesEmptyTextAndOrderedStructureValues() { + String detail = ""; + String data = """ + +1.2300e+2truedark.red +