From 5da2e6f2975bdba1a851c8ad3deb5df74c83ffab Mon Sep 17 00:00:00 2001 From: Karl Kauc Date: Fri, 15 May 2026 20:04:56 +0200 Subject: [PATCH] Phase 6 (final): data binding + JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Data_Binding_JSON/: FundsXML<->JSON converter (Python, stdlib+lxml) with to-json/from-json/roundtrip; from-json emits XSD-valid FundsXML, round-trip preserves NAV/position-count/percentage (verified). NativeBinding.java binds FundsXML to Java records via DOM/XPath, NO JAXB (verified on 4.2.9 and 4.0.0). README documents the generated-binding alternatives (JAXB xjc, xsdata, xsd.exe) against the official fetched schema. CI gains a JSON round-trip (XSD-validated, figures asserted) + native Java binding step (verified locally). Top-level index updated — roadmap complete. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 20 ++ .gitignore | 3 + Data_Binding_JSON/README.md | 51 +++++ Data_Binding_JSON/java/NativeBinding.java | 98 +++++++++ Data_Binding_JSON/python/fundsxml_json.py | 241 ++++++++++++++++++++++ README.md | 1 + 6 files changed, 414 insertions(+) create mode 100644 Data_Binding_JSON/README.md create mode 100644 Data_Binding_JSON/java/NativeBinding.java create mode 100644 Data_Binding_JSON/python/fundsxml_json.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b24fd3a..678b27c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,6 +157,26 @@ jobs: xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd chunks/chunk-0001.xml python3 $P/delta_diff.py big.xml big.xml # identical -> exit 0 + - name: Data binding / JSON - round-trip + native Java binding + run: | + set -e + python3 -m pip install --quiet lxml + SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml + python3 Data_Binding_JSON/python/fundsxml_json.py roundtrip "$SRC" rj.xml + xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd rj.xml + python3 - "$SRC" rj.xml <<'PY' + import re, sys + o, r = (open(p).read() for p in sys.argv[1:3]) + nav = lambda x: re.search(r'\s*([0-9.]+)', x).group(1) + npos = lambda x: len(re.findall(r'', x)) + assert abs(float(nav(o)) - float(nav(r))) < 0.01, (nav(o), nav(r)) + assert npos(o) == npos(r), (npos(o), npos(r)) + print("JSON round-trip preserved:", nav(r), npos(r)) + PY + javac -d /tmp/nb Data_Binding_JSON/java/NativeBinding.java + java -cp /tmp/nb NativeBinding "$SRC" | tee nb.txt + grep -q '^Positions : 21$' nb.txt + - name: Regression - legacy XSLT 1.0 report still runs run: | xsltproc XSLT_DataQuality_Checks/Enhanced_Check/FundsXML_CompleteDQReport_HTML.xsl \ diff --git a/.gitignore b/.gitignore index 6e7c4ba..47400de 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,8 @@ big.xml chunks/ agg.txt aggj.txt +nb.txt +rj.xml +fund.json *.db *.svrl diff --git a/Data_Binding_JSON/README.md b/Data_Binding_JSON/README.md new file mode 100644 index 0000000..146fce9 --- /dev/null +++ b/Data_Binding_JSON/README.md @@ -0,0 +1,51 @@ +# Data Binding & JSON + +![status](https://img.shields.io/badge/JSON%20round--trip%20%2B%20Java%20binding-verified-brightgreen) + +Two recurring enterprise needs: exposing FundsXML to **JSON** APIs, and +**binding** FundsXML to typed objects. + +## FundsXML ⇄ JSON (verified) + +[`python/fundsxml_json.py`](python/fundsxml_json.py) — `to-json` / `from-json` +/ `roundtrip` (stdlib + lxml). Stable JSON shape +(`document` / `fund` / `shareClasses` / `positions` / `assets`); `from-json` +produces **XSD-valid** FundsXML 4.2.9. Lossless for the positions core, +intentionally lossy for issuer / derivative / regulatory detail (same scope +boundary as `Database_Integration/`). + +```bash +python3 Data_Binding_JSON/python/fundsxml_json.py to-json \ + FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml fund.json +python3 Data_Binding_JSON/python/fundsxml_json.py roundtrip \ + FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml regenerated.xml +xmllint --noout --schema .schema-cache/4.2.9/FundsXML.xsd regenerated.xml +``` + +Verified: NAV, position count and percentage-sum preserved through +XML → JSON → XML. + +## Binding: native vs. generated + +[`java/NativeBinding.java`](java/NativeBinding.java) — **native Java, no JAXB**: +a thin hand-written binding over DOM + XPath into Java `record`s. Verified on +4.2.9 and 4.0.0 (FundsXML 4.x is backward compatible, so one binding spans +versions). This is the recommended default: the FundsXML schema is very large, +so a full generated model is heavy and brittle to maintain. + +### Generated-binding references (when you do want codegen) + +| Stack | Tool | Command (against the fetched schema) | +|-------|------|--------------------------------------| +| Java | JAXB `xjc` | `xjc -d src -p org.fundsxml.model .schema-cache/4.2.9/FundsXML.xsd` | +| Python | `xsdata` | `xsdata --package fundsxml.model .schema-cache/4.2.9/FundsXML.xsd` | +| .NET | `xsd.exe` / `XmlSerializer` | `xsd.exe /classes /namespace:FundsXml.Model .schema-cache\4.2.9\FundsXML.xsd` | + +All three consume the **official released schema** fetched by +`tools/fetch-schema.sh` (which also pulls the imported `xmldsig-core-schema.xsd` +for 4.2.9). Trade-off: generated models are type-safe but regenerate on every +schema bump and produce thousands of classes; the native binding stays small +and version-tolerant. Pick per use case. + +CI runs the JSON round-trip (XSD-validated, figures asserted) and the native +Java binding on every push. diff --git a/Data_Binding_JSON/java/NativeBinding.java b/Data_Binding_JSON/java/NativeBinding.java new file mode 100644 index 0000000..36b30ca --- /dev/null +++ b/Data_Binding_JSON/java/NativeBinding.java @@ -0,0 +1,98 @@ +// Native Java data binding for FundsXML — DOM + XPath into records, NO JAXB. +// +// javac -d /tmp/nb Data_Binding_JSON/java/NativeBinding.java +// java -cp /tmp/nb NativeBinding +// +// Why no JAXB: the FundsXML schema is huge; generating/maintaining a full JAXB +// model is heavy and brittle across versions. For most integration work a thin +// hand-written binding over DOM/XPath is simpler, version-tolerant (FundsXML +// 4.x is backward compatible) and dependency-free. The codegen alternatives +// (JAXB xjc, xsdata, xsd.exe) are documented in the README. +// +// FundsXML 4.x has no XML namespace — XPath uses bare element names. +// Security: namespace-unaware is fine here; DTDs/external entities disabled. + +import java.io.FileInputStream; +import java.util.ArrayList; +import java.util.List; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +public class NativeBinding { + + // Immutable binding types (Java records). + record Fund(String lei, String name, String currency, double totalNav, + int positionCount) {} + record Position(String uniqueId, String isin, double value, + double percentage, String kind) {} + + public static void main(String[] args) throws Exception { + if (args.length != 1) { + System.err.println("usage: NativeBinding "); + System.exit(2); + } + + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + dbf.setExpandEntityReferences(false); + Document doc; + try (FileInputStream in = new FileInputStream(args[0])) { + doc = dbf.newDocumentBuilder().parse(in); + } + XPath xp = XPathFactory.newInstance().newXPath(); + + String ccy = xp.evaluate("/FundsXML4/Funds/Fund/Currency", doc); + Fund fund = new Fund( + xp.evaluate("/FundsXML4/Funds/Fund/Identifiers/LEI", doc), + xp.evaluate("/FundsXML4/Funds/Fund/Names/OfficialName", doc), + ccy, + Double.parseDouble(xp.evaluate( + "/FundsXML4/Funds/Fund/FundDynamicData/TotalAssetValues/" + + "TotalAssetValue/TotalNetAssetValue/Amount[@ccy='" + ccy + + "']", doc)), + ((Number) xp.evaluate( + "count(//Position)", doc, XPathConstants.NUMBER)).intValue()); + + NodeList pn = (NodeList) xp.evaluate("//Positions/Position", doc, + XPathConstants.NODESET); + List positions = new ArrayList<>(); + double sumPct = 0; + for (int i = 0; i < pn.getLength(); i++) { + Node p = pn.item(i); + String kind = xp.evaluate( + "local-name(*[local-name()='Equity' or local-name()='Bond' " + + "or local-name()='ShareClass' or local-name()='Warrant' " + + "or local-name()='Certificate' or local-name()='Option' " + + "or local-name()='Future' or local-name()='FXForward' " + + "or local-name()='Swap' or local-name()='Repo' " + + "or local-name()='RealEstate' or local-name()='CallMoney'][1])", + p); + double pct = num(xp.evaluate("TotalPercentage", p)); + sumPct += pct; + positions.add(new Position( + xp.evaluate("UniqueID", p), + xp.evaluate("Identifiers/ISIN", p), + num(xp.evaluate("TotalValue/Amount[@ccy='" + ccy + "']", p)), + pct, kind)); + } + + System.out.println("Fund : " + fund.name()); + System.out.println("LEI / ccy : " + fund.lei() + " / " + fund.currency()); + System.out.printf("Total NAV : %.2f%n", fund.totalNav()); + System.out.println("Positions : " + fund.positionCount()); + System.out.printf("Sum %% : %.4f%n", sumPct); + System.out.println("First 3 :"); + positions.stream().limit(3).forEach(p -> System.out.printf( + " %s %-12s %14.2f %6.2f%% [%s]%n", + p.uniqueId(), p.isin(), p.value(), p.percentage(), p.kind())); + } + + private static double num(String s) { + return (s == null || s.isBlank()) ? 0.0 : Double.parseDouble(s.trim()); + } +} diff --git a/Data_Binding_JSON/python/fundsxml_json.py b/Data_Binding_JSON/python/fundsxml_json.py new file mode 100644 index 0000000..df351e6 --- /dev/null +++ b/Data_Binding_JSON/python/fundsxml_json.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""FundsXML <-> JSON — runnable, verified converter (stdlib + lxml). + +Many modern APIs speak JSON while the system of record is FundsXML. This maps +the positions core to a clean, stable JSON shape and back to XSD-valid +FundsXML. + + to-json + from-json + roundtrip # xml -> json -> xml in one shot + +JSON shape (lossless for the positions core; intentionally lossy for issuer / +derivative / regulatory detail — same scope boundary as Database_Integration/): + + {"document": {...}, "fund": {...}, "shareClasses": [...], + "positions": [...], "assets": [...]} + +FundsXML 4.x has no XML namespace. +""" +import json +import sys +import tempfile +from pathlib import Path + +from lxml import etree + +SCHEMA = ("https://github.com/fundsxml/schema/releases/download/" + "4.2.9/FundsXML.xsd") +XSI = "http://www.w3.org/2001/XMLSchema-instance" +POSITION_KINDS = {"Equity", "Bond", "ShareClass", "Warrant", "Certificate", + "Option", "Future", "FXForward", "Swap", "Repo", + "RealEstate", "CallMoney", "Account", "Generic"} +QTY_ELEM = {"Equity": "Units", "Warrant": "Units", "Certificate": "Units", + "Bond": "Nominal", "ShareClass": "Shares", + "Option": "Contracts", "Future": "Contracts"} + + +def _t(node, path): + r = node.xpath(path) + return r[0].text if r and r[0].text is not None else None + + +def _f(v): + return float(v) if v not in (None, "") else None + + +# --------------------------------------------------------------- to JSON ----- +def to_json(xml_path: str) -> dict: + doc = etree.parse(xml_path) + cd = doc.xpath("/FundsXML4/ControlData")[0] + fund = doc.xpath("/FundsXML4/Funds/Fund")[0] + ccy = _t(fund, "Currency") + tav = fund.xpath("FundDynamicData/TotalAssetValues/TotalAssetValue")[0] + + out = { + "document": { + "id": _t(cd, "UniqueDocumentID"), + "generated": _t(cd, "DocumentGenerated"), + "version": _t(cd, "Version"), + "contentDate": _t(cd, "ContentDate"), + "dataOperation": _t(cd, "DataOperation"), + }, + "fund": { + "lei": _t(fund, "Identifiers/LEI"), + "officialName": _t(fund, "Names/OfficialName"), + "currency": ccy, + "navDate": _t(tav, "NavDate"), + "totalNav": _f(tav.xpath( + f"TotalNetAssetValue/Amount[@ccy='{ccy}']")[0].text), + }, + "shareClasses": [], + "positions": [], + "assets": [], + } + for sc in fund.xpath("SingleFund/ShareClasses/ShareClass"): + out["shareClasses"].append({ + "isin": _t(sc, "Identifiers/ISIN"), + "officialName": _t(sc, "Names/OfficialName"), + "currency": _t(sc, "Currency"), + "navPrice": _f(_t(sc, "Prices/Price/NavPrice")), + "navFundCcy": _f((sc.xpath( + "TotalAssetValues/TotalAssetValue/TotalNetAssetValue/" + f"Amount[@ccy='{ccy}']") or [None])[0].text + if sc.xpath("TotalAssetValues/TotalAssetValue/" + f"TotalNetAssetValue/Amount[@ccy='{ccy}']") + else None), + "sharesOutstanding": _f(_t( + sc, "TotalAssetValues/TotalAssetValue/SharesOutstanding")), + }) + for a in doc.xpath("/FundsXML4/AssetMasterData/Asset"): + out["assets"].append({ + "uniqueId": _t(a, "UniqueID"), + "isin": _t(a, "Identifiers/ISIN"), + "name": _t(a, "Name"), + "assetType": _t(a, "AssetType"), + "currency": _t(a, "Currency"), + "country": _t(a, "Country"), + }) + for p in fund.xpath( + "FundDynamicData/Portfolios/Portfolio/Positions/Position"): + kinds = [c.tag for c in p if c.tag in POSITION_KINDS] + kind = kinds[0] if kinds else None + out["positions"].append({ + "uniqueId": _t(p, "UniqueID"), + "isin": _t(p, "Identifiers/ISIN"), + "currency": _t(p, "Currency"), + "value": _f(p.xpath(f"TotalValue/Amount[@ccy='{ccy}']")[0].text), + "percentage": _f(_t(p, "TotalPercentage")), + "kind": kind, + "kindQty": _f(_t(p, f"{kind}/{QTY_ELEM[kind]}")) + if kind in QTY_ELEM else None, + }) + return out + + +# ------------------------------------------------------------- from JSON ----- +def _el(parent, tag, text=None, **attrs): + e = etree.SubElement(parent, tag) + for k, v in attrs.items(): + e.set(k, str(v)) + if text is not None: + e.text = str(text) + return e + + +def from_json(data: dict, out_path: str) -> None: + d, fu = data["document"], data["fund"] + ccy = fu["currency"] + root = etree.Element("FundsXML4", nsmap={"xsi": XSI}) + root.set(f"{{{XSI}}}noNamespaceSchemaLocation", SCHEMA) + + cd = _el(root, "ControlData") + _el(cd, "UniqueDocumentID", d["id"]) + _el(cd, "DocumentGenerated", d.get("generated") or "2025-10-02T00:00:00") + if d.get("version"): + _el(cd, "Version", d["version"]) + _el(cd, "ContentDate", d["contentDate"]) + ds = _el(cd, "DataSupplier") + _el(ds, "SystemCountry", "AT") + _el(ds, "Short", "EURAM") + _el(ds, "Name", "Erste Asset Management GmbH") + _el(ds, "Type", "Asset Manager") + _el(cd, "DataOperation", d.get("dataOperation") or "INITIAL") + + fund = _el(_el(root, "Funds"), "Fund") + if fu.get("lei"): + _el(_el(fund, "Identifiers"), "LEI", fu["lei"]) + _el(_el(fund, "Names"), "OfficialName", fu["officialName"]) + _el(fund, "Currency", ccy) + _el(fund, "SingleFundFlag", "true") + fdd = _el(fund, "FundDynamicData") + tav = _el(_el(_el(fdd, "TotalAssetValues"), "TotalAssetValue"), + "NavDate", fu["navDate"]).getparent() + _el(tav, "TotalAssetNature", "OFFICIAL") + _el(_el(tav, "TotalNetAssetValue"), "Amount", + f'{fu["totalNav"]:.2f}', ccy=ccy) + port = _el(_el(fdd, "Portfolios"), "Portfolio") + _el(port, "NavDate", fu["navDate"]) + poss = _el(port, "Positions") + for p in data["positions"]: + pos = _el(poss, "Position") + _el(pos, "UniqueID", p["uniqueId"]) + if p.get("isin"): + _el(_el(pos, "Identifiers"), "ISIN", p["isin"]) + if p.get("currency"): + _el(pos, "Currency", p["currency"]) + _el(_el(pos, "TotalValue"), "Amount", f'{p["value"]:.2f}', ccy=ccy) + _el(pos, "TotalPercentage", f'{p["percentage"]:.2f}') + kind = p["kind"] if p.get("kind") in POSITION_KINDS else "Generic" + ke = _el(pos, kind) + if kind in QTY_ELEM and p.get("kindQty") is not None: + _el(ke, QTY_ELEM[kind], f'{p["kindQty"]:.2f}') + + if data.get("shareClasses"): + sce = _el(_el(fund, "SingleFund"), "ShareClasses") + for sc in data["shareClasses"]: + x = _el(sce, "ShareClass") + _el(_el(x, "Identifiers"), "ISIN", sc["isin"]) + if sc.get("officialName"): + _el(_el(x, "Names"), "OfficialName", sc["officialName"]) + _el(x, "Currency", sc["currency"]) + if sc.get("navPrice") is not None: + pr = _el(_el(x, "Prices"), "Price") + _el(pr, "ActionCode", "C") + _el(pr, "NavDate", fu["navDate"]) + _el(pr, "PriceCurrency", sc["currency"]) + _el(pr, "PriceNature", "OFFICIAL") + _el(pr, "NavPrice", f'{sc["navPrice"]:.2f}') + if sc.get("navFundCcy") is not None: + t = _el(_el(x, "TotalAssetValues"), "TotalAssetValue") + _el(t, "NavDate", fu["navDate"]) + _el(t, "TotalAssetNature", "OFFICIAL") + _el(_el(t, "TotalNetAssetValue"), "Amount", + f'{sc["navFundCcy"]:.2f}', ccy=ccy) + if sc.get("sharesOutstanding") is not None: + _el(t, "SharesOutstanding", + f'{sc["sharesOutstanding"]:.0f}') + + if data.get("assets"): + amd = _el(root, "AssetMasterData") + for a in data["assets"]: + ae = _el(amd, "Asset") + _el(ae, "UniqueID", a["uniqueId"]) + if a.get("isin"): + _el(_el(ae, "Identifiers"), "ISIN", a["isin"]) + _el(ae, "Currency", a.get("currency") or ccy) + if a.get("country"): + _el(ae, "Country", a["country"]) + _el(ae, "Name", a["name"]) + _el(ae, "AssetType", a["assetType"]) + + etree.ElementTree(root).write(out_path, xml_declaration=True, + encoding="UTF-8", pretty_print=True) + + +def main() -> int: + a = sys.argv[1:] + if len(a) != 3: + print(__doc__, file=sys.stderr) + return 2 + mode, src, dst = a + if mode == "to-json": + Path(dst).write_text(json.dumps(to_json(src), indent=2)) + print("wrote", dst) + elif mode == "from-json": + from_json(json.loads(Path(src).read_text()), dst) + print("wrote", dst) + elif mode == "roundtrip": + with tempfile.TemporaryDirectory() as t: + j = Path(t) / "doc.json" + j.write_text(json.dumps(to_json(src), indent=2)) + from_json(json.loads(j.read_text()), dst) + print(f"round-trip ok: {src} -> JSON -> {dst}") + else: + print(f"unknown mode {mode!r}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/README.md b/README.md index bccd22f..3c3cb22 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ locations. Items marked _(planned)_ are on the roadmap (see | XML signature sign/verify | Apache Santuario (Java), .NET, xmlsec1, signxml | [XML_Signature/](./XML_Signature/) | ✅ | | Database load ↔ generate | SQLite (verified) + Oracle/SQL Server/Postgres (code) | [Database_Integration/](./Database_Integration/) | ✅ | | Large-file / streaming | lxml iterparse + Java StAX, split, delta-diff | [Large_File_Processing/](./Large_File_Processing/) | ✅ | +| Data binding & JSON | FundsXML⇄JSON, native Java binding, codegen refs | [Data_Binding_JSON/](./Data_Binding_JSON/) | ✅ | ## Repository Structure