From 348847c1ad80a86a33204916f28931d0ea0912a6 Mon Sep 17 00:00:00 2001 From: Karl Kauc Date: Fri, 15 May 2026 21:44:45 +0200 Subject: [PATCH] Database_Integration: standalone multi-language, multi-fund examples Rework Database_Integration so a developer finds a runnable, heavily-commented example in their own language for BOTH directions (FundsXML -> DB, DB -> FundsXML), handling multiple nodes (>=2 Funds, multi-Portfolio/Position/ ShareClass/Asset). - ddl/schema.sql: redesigned multi-fund model (document -> fund -> portfolio -> position; share_class per fund; document-scoped asset) with 1-based ordinals so order/duplicates round-trip exactly. - Standalone same-CLI programs (init/import/export/roundtrip), all verified locally to produce import==export: python/fundsxml_db.py (stdlib sqlite3 + lxml) java/FundsXmlDb.java (sqlite-jdbc, native javax.xml, no JAXB) javascript/fundsxml_db.mjs (sql.js + @xmldom/xmldom + xpath) csharp/FundsXmlDb.cs(+csproj) (Microsoft.Data.Sqlite) - tools/xml_equiv.py: shared normalized comparator proving import file == export file (ignores DocumentGenerated timestamp + numeric/whitespace/attr ordering; child order significant); always paired with XSD validation. - FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml: round-trip-faithful multi-fund fixture (2 funds, one with 2 portfolios), XSD-valid. - Oracle/SQLServer/Postgres load+generate SQL updated to the multi-fund nested-XMLTABLE / grouped-XMLAGG / FOR XML pattern (code references). - Data_Binding_JSON/python/fundsxml_json.py reworked to the multi-fund shape (lossless -> xml_equiv-equal on the fixture). - tools/fetch-tools.sh fetches sqlite-jdbc; CI runs the multi-fund round-trip in all 4 languages (equivalence + XSD + cross-language) plus the single-fund sample; setup-node and setup-dotnet added. .gitignore: node_modules, bin/ obj/, macOS metadata. Verified locally: every language's export is xml_equiv-equal to the fixture and to each other, and XSD-valid; Mixed-Fund (n=1) still round-trips. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 69 ++- .gitignore | 12 + Data_Binding_JSON/python/fundsxml_json.py | 276 +++++----- Database_Integration/README.md | 112 ++-- Database_Integration/csharp/FundsXmlDb.cs | 506 ++++++++++++++++++ Database_Integration/csharp/FundsXmlDb.csproj | 16 + Database_Integration/ddl/schema.sql | 143 +++-- .../generate_fundsxml/oracle.sql | 128 +++-- .../generate_fundsxml/postgres.sql | 134 +++-- .../generate_fundsxml/sqlserver.sql | 96 ++-- Database_Integration/java/FundsXmlDb.java | 493 +++++++++++++++++ .../javascript/fundsxml_db.mjs | 317 +++++++++++ Database_Integration/javascript/package.json | 16 + .../load_from_fundsxml/oracle.sql | 80 +-- .../load_from_fundsxml/postgres.sql | 97 ++-- .../load_from_fundsxml/sqlserver.sql | 83 +-- Database_Integration/python/fundsxml_db.py | 486 ++++++++++------- Database_Integration/tools/xml_equiv.py | 110 ++++ .../4.2.9/positions/Multi-Fund_Positions.xml | 276 ++++++++++ README.md | 2 +- tools/fetch-tools.sh | 6 + 21 files changed, 2777 insertions(+), 681 deletions(-) create mode 100644 Database_Integration/csharp/FundsXmlDb.cs create mode 100644 Database_Integration/csharp/FundsXmlDb.csproj create mode 100644 Database_Integration/java/FundsXmlDb.java create mode 100644 Database_Integration/javascript/fundsxml_db.mjs create mode 100644 Database_Integration/javascript/package.json create mode 100644 Database_Integration/tools/xml_equiv.py create mode 100644 FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 678b27c..c4e0b04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,14 @@ jobs: with: python-version: "3.11" + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: "8.0" + - name: Fetch official schemas (4.2.9, 4.1.0, 4.0.0) run: | for v in 4.2.9 4.1.0 4.0.0; do tools/fetch-schema.sh "$v"; done @@ -120,24 +128,50 @@ jobs: fi echo "tamper correctly detected" - - name: DB integration - FundsXML -> SQLite -> FundsXML round-trip + - name: DB integration - multi-fund round-trip in 4 languages run: | set -e python3 -m pip install --quiet lxml - SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml - python3 Database_Integration/python/fundsxml_db.py roundtrip "$SRC" regen.xml - xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd regen.xml - python3 - "$SRC" regen.xml <<'PY' - import re, sys - o, r = (open(p).read() for p in sys.argv[1:3]) - nav = lambda x: re.search(r'\s*([0-9.]+)', x).group(1) - npos = lambda x: len(re.findall(r'', x)) - spct = lambda x: round(sum(float(v) for v in re.findall(r'([0-9.]+)', x)), 2) - assert abs(float(nav(o)) - float(nav(r))) < 0.01, (nav(o), nav(r)) - assert npos(o) == npos(r), (npos(o), npos(r)) - assert abs(spct(o) - spct(r)) < 0.01, (spct(o), spct(r)) - print("round-trip figures preserved:", nav(r), npos(r), spct(r)) - PY + tools/fetch-tools.sh # sqlite-jdbc -> .lib/ + FX=FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml + MX=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml + EQ="python3 Database_Integration/tools/xml_equiv.py" + XSD=".schema-cache/4.2.9/FundsXML.xsd" + + # The multi-fund fixture must itself be schema-valid. + xmllint --noout --nonet --schema "$XSD" "$FX" + + # Python + python3 Database_Integration/python/fundsxml_db.py roundtrip "$FX" py.xml + xmllint --noout --nonet --schema "$XSD" py.xml + $EQ "$FX" py.xml + + # Java (sqlite-jdbc) + CP=.lib/sqlite-jdbc-3.46.1.3.jar + javac -cp "$CP" -d /tmp/jdb Database_Integration/java/FundsXmlDb.java + java --enable-native-access=ALL-UNNAMED -cp "$CP:/tmp/jdb" \ + FundsXmlDb roundtrip "$FX" java.xml + xmllint --noout --nonet --schema "$XSD" java.xml + $EQ "$FX" java.xml + + # JavaScript (sql.js) + ( cd Database_Integration/javascript && npm install --no-audit --no-fund ) + node Database_Integration/javascript/fundsxml_db.mjs roundtrip "$FX" js.xml + xmllint --noout --nonet --schema "$XSD" js.xml + $EQ "$FX" js.xml + + # C# (.NET) + dotnet run --project Database_Integration/csharp -- roundtrip "$FX" cs.xml + xmllint --noout --nonet --schema "$XSD" cs.xml + $EQ "$FX" cs.xml + + # Cross-language: every export identical to Python's. + $EQ py.xml java.xml && $EQ py.xml js.xml && $EQ py.xml cs.xml + + # Single-fund sample still round-trips (lossy -> XSD-valid only). + python3 Database_Integration/python/fundsxml_db.py roundtrip "$MX" mx.xml + xmllint --noout --nonet --schema "$XSD" mx.xml + echo "multi-fund round-trip equivalent in python/java/javascript/csharp" - name: Large-file - streaming aggregate / split / delta (constant memory) run: | @@ -164,6 +198,11 @@ jobs: SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml python3 Data_Binding_JSON/python/fundsxml_json.py roundtrip "$SRC" rj.xml xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd rj.xml + # Multi-fund JSON round-trip is lossless -> must be xml_equiv-equal. + MF=FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml + python3 Data_Binding_JSON/python/fundsxml_json.py roundtrip "$MF" rjm.xml + xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd rjm.xml + python3 Database_Integration/tools/xml_equiv.py "$MF" rjm.xml python3 - "$SRC" rj.xml <<'PY' import re, sys o, r = (open(p).read() for p in sys.argv[1:3]) diff --git a/.gitignore b/.gitignore index 47400de..63d49dd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,9 @@ CLAUDE.md +# macOS filesystem metadata (the working copy is on a WebDAV mount) +.DS_Store +._* + # Locally fetched XSD releases (via tools/fetch-schema.sh) — not committed, # the source of truth is the official GitHub release. .schema-cache/ @@ -33,3 +37,11 @@ rj.xml fund.json *.db *.svrl + +# Node example deps (Database_Integration/javascript) — restored via npm install +node_modules/ +package-lock.json + +# .NET build output (Database_Integration/csharp) — restored via dotnet build +bin/ +obj/ diff --git a/Data_Binding_JSON/python/fundsxml_json.py b/Data_Binding_JSON/python/fundsxml_json.py index df351e6..043d5cf 100644 --- a/Data_Binding_JSON/python/fundsxml_json.py +++ b/Data_Binding_JSON/python/fundsxml_json.py @@ -1,22 +1,27 @@ #!/usr/bin/env python3 -"""FundsXML <-> JSON — runnable, verified converter (stdlib + lxml). - -Many modern APIs speak JSON while the system of record is FundsXML. This maps -the positions core to a clean, stable JSON shape and back to XSD-valid -FundsXML. - - to-json - from-json - roundtrip # xml -> json -> xml in one shot - -JSON shape (lossless for the positions core; intentionally lossy for issuer / -derivative / regulatory detail — same scope boundary as Database_Integration/): - - {"document": {...}, "fund": {...}, "shareClasses": [...], - "positions": [...], "assets": [...]} - -FundsXML 4.x has no XML namespace. -""" +# ============================================================================= +# FundsXML <-> JSON — runnable reference (Python 3, stdlib + lxml). +# +# Many systems speak JSON internally while exchanging FundsXML externally. This +# shows a faithful, MULTI-FUND JSON projection and back to XSD-valid FundsXML. +# +# to-json +# from-json +# roundtrip # xml -> json -> xml in one shot +# +# JSON shape (multi-fund; mirrors Database_Integration/ddl/schema.sql): +# { +# "document": {... ControlData ...}, +# "funds": [ { ... , "shareClasses":[...], +# "portfolios":[ { "positions":[...] } ] } ], +# "assets": [ ... document-scoped AssetMasterData ... ] +# } +# +# Lossless for the captured model -> a round-trip-faithful file compares EQUAL +# with Database_Integration/tools/xml_equiv.py (ignoring the volatile +# DocumentGenerated timestamp and numeric/whitespace formatting). FundsXML 4.x +# has no XML namespace; the parser disables DTD/external entities (XXE-safe). +# ============================================================================= import json import sys import tempfile @@ -27,6 +32,8 @@ SCHEMA = ("https://github.com/fundsxml/schema/releases/download/" "4.2.9/FundsXML.xsd") XSI = "http://www.w3.org/2001/XMLSchema-instance" +# Position instrument-class element + its mandatory quantity child (see the +# Database_Integration examples for the rationale). POSITION_KINDS = {"Equity", "Bond", "ShareClass", "Warrant", "Certificate", "Option", "Future", "FXForward", "Swap", "Repo", "RealEstate", "CallMoney", "Account", "Generic"} @@ -44,14 +51,12 @@ def _f(v): return float(v) if v not in (None, "") else None -# --------------------------------------------------------------- to JSON ----- +# ---------------------------------------------------------------- to JSON ---- def to_json(xml_path: str) -> dict: - doc = etree.parse(xml_path) + p = etree.XMLParser(resolve_entities=False, no_network=True, + load_dtd=False) + doc = etree.parse(xml_path, p) cd = doc.xpath("/FundsXML4/ControlData")[0] - fund = doc.xpath("/FundsXML4/Funds/Fund")[0] - ccy = _t(fund, "Currency") - tav = fund.xpath("FundDynamicData/TotalAssetValues/TotalAssetValue")[0] - out = { "document": { "id": _t(cd, "UniqueDocumentID"), @@ -59,34 +64,60 @@ def to_json(xml_path: str) -> dict: "version": _t(cd, "Version"), "contentDate": _t(cd, "ContentDate"), "dataOperation": _t(cd, "DataOperation"), + "supplier": { + "country": _t(cd, "DataSupplier/SystemCountry"), + "short": _t(cd, "DataSupplier/Short"), + "name": _t(cd, "DataSupplier/Name"), + "type": _t(cd, "DataSupplier/Type"), + }, }, - "fund": { + "funds": [], + "assets": [], + } + for fund in doc.xpath("/FundsXML4/Funds/Fund"): # every fund + ccy = _t(fund, "Currency") + tav = fund.xpath("FundDynamicData/TotalAssetValues/TotalAssetValue")[0] + fj = { "lei": _t(fund, "Identifiers/LEI"), "officialName": _t(fund, "Names/OfficialName"), "currency": ccy, + "singleFundFlag": _t(fund, "SingleFundFlag"), "navDate": _t(tav, "NavDate"), "totalNav": _f(tav.xpath( f"TotalNetAssetValue/Amount[@ccy='{ccy}']")[0].text), - }, - "shareClasses": [], - "positions": [], - "assets": [], - } - for sc in fund.xpath("SingleFund/ShareClasses/ShareClass"): - out["shareClasses"].append({ - "isin": _t(sc, "Identifiers/ISIN"), - "officialName": _t(sc, "Names/OfficialName"), - "currency": _t(sc, "Currency"), - "navPrice": _f(_t(sc, "Prices/Price/NavPrice")), - "navFundCcy": _f((sc.xpath( - "TotalAssetValues/TotalAssetValue/TotalNetAssetValue/" - f"Amount[@ccy='{ccy}']") or [None])[0].text - if sc.xpath("TotalAssetValues/TotalAssetValue/" + "shareClasses": [], + "portfolios": [], + } + for sc in fund.xpath("SingleFund/ShareClasses/ShareClass"): + navf = sc.xpath("TotalAssetValues/TotalAssetValue/" f"TotalNetAssetValue/Amount[@ccy='{ccy}']") - else None), - "sharesOutstanding": _f(_t( - sc, "TotalAssetValues/TotalAssetValue/SharesOutstanding")), - }) + fj["shareClasses"].append({ + "isin": _t(sc, "Identifiers/ISIN"), + "officialName": _t(sc, "Names/OfficialName"), + "currency": _t(sc, "Currency"), + "navPrice": _f(_t(sc, "Prices/Price/NavPrice")), + "navFundCcy": _f(navf[0].text) if navf else None, + "sharesOutstanding": _f(_t( + sc, "TotalAssetValues/TotalAssetValue/SharesOutstanding")), + }) + for port in fund.xpath("FundDynamicData/Portfolios/Portfolio"): + pj = {"navDate": _t(port, "NavDate"), "positions": []} + for pos in port.xpath("Positions/Position"): + kinds = [c.tag for c in pos if c.tag in POSITION_KINDS] + kind = kinds[0] if kinds else None + pj["positions"].append({ + "uniqueId": _t(pos, "UniqueID"), + "isin": _t(pos, "Identifiers/ISIN"), + "currency": _t(pos, "Currency"), + "value": _f(pos.xpath( + f"TotalValue/Amount[@ccy='{ccy}']")[0].text), + "percentage": _f(_t(pos, "TotalPercentage")), + "kind": kind, + "kindQty": _f(_t(pos, f"{kind}/{QTY_ELEM[kind]}")) + if kind in QTY_ELEM else None, + }) + fj["portfolios"].append(pj) + out["funds"].append(fj) for a in doc.xpath("/FundsXML4/AssetMasterData/Asset"): out["assets"].append({ "uniqueId": _t(a, "UniqueID"), @@ -96,24 +127,10 @@ def to_json(xml_path: str) -> dict: "currency": _t(a, "Currency"), "country": _t(a, "Country"), }) - for p in fund.xpath( - "FundDynamicData/Portfolios/Portfolio/Positions/Position"): - kinds = [c.tag for c in p if c.tag in POSITION_KINDS] - kind = kinds[0] if kinds else None - out["positions"].append({ - "uniqueId": _t(p, "UniqueID"), - "isin": _t(p, "Identifiers/ISIN"), - "currency": _t(p, "Currency"), - "value": _f(p.xpath(f"TotalValue/Amount[@ccy='{ccy}']")[0].text), - "percentage": _f(_t(p, "TotalPercentage")), - "kind": kind, - "kindQty": _f(_t(p, f"{kind}/{QTY_ELEM[kind]}")) - if kind in QTY_ELEM else None, - }) return out -# ------------------------------------------------------------- from JSON ----- +# -------------------------------------------------------------- from JSON ---- def _el(parent, tag, text=None, **attrs): e = etree.SubElement(parent, tag) for k, v in attrs.items(): @@ -124,77 +141,84 @@ def _el(parent, tag, text=None, **attrs): def from_json(data: dict, out_path: str) -> None: - d, fu = data["document"], data["fund"] - ccy = fu["currency"] + dd = data["document"] root = etree.Element("FundsXML4", nsmap={"xsi": XSI}) root.set(f"{{{XSI}}}noNamespaceSchemaLocation", SCHEMA) cd = _el(root, "ControlData") - _el(cd, "UniqueDocumentID", d["id"]) - _el(cd, "DocumentGenerated", d.get("generated") or "2025-10-02T00:00:00") - if d.get("version"): - _el(cd, "Version", d["version"]) - _el(cd, "ContentDate", d["contentDate"]) + _el(cd, "UniqueDocumentID", dd["id"]) + _el(cd, "DocumentGenerated", dd.get("generated") or "2025-10-02T00:00:00") + if dd.get("version"): + _el(cd, "Version", dd["version"]) + _el(cd, "ContentDate", dd["contentDate"]) + s = dd["supplier"] ds = _el(cd, "DataSupplier") - _el(ds, "SystemCountry", "AT") - _el(ds, "Short", "EURAM") - _el(ds, "Name", "Erste Asset Management GmbH") - _el(ds, "Type", "Asset Manager") - _el(cd, "DataOperation", d.get("dataOperation") or "INITIAL") - - fund = _el(_el(root, "Funds"), "Fund") - if fu.get("lei"): - _el(_el(fund, "Identifiers"), "LEI", fu["lei"]) - _el(_el(fund, "Names"), "OfficialName", fu["officialName"]) - _el(fund, "Currency", ccy) - _el(fund, "SingleFundFlag", "true") - fdd = _el(fund, "FundDynamicData") - tav = _el(_el(_el(fdd, "TotalAssetValues"), "TotalAssetValue"), - "NavDate", fu["navDate"]).getparent() - _el(tav, "TotalAssetNature", "OFFICIAL") - _el(_el(tav, "TotalNetAssetValue"), "Amount", - f'{fu["totalNav"]:.2f}', ccy=ccy) - port = _el(_el(fdd, "Portfolios"), "Portfolio") - _el(port, "NavDate", fu["navDate"]) - poss = _el(port, "Positions") - for p in data["positions"]: - pos = _el(poss, "Position") - _el(pos, "UniqueID", p["uniqueId"]) - if p.get("isin"): - _el(_el(pos, "Identifiers"), "ISIN", p["isin"]) - if p.get("currency"): - _el(pos, "Currency", p["currency"]) - _el(_el(pos, "TotalValue"), "Amount", f'{p["value"]:.2f}', ccy=ccy) - _el(pos, "TotalPercentage", f'{p["percentage"]:.2f}') - kind = p["kind"] if p.get("kind") in POSITION_KINDS else "Generic" - ke = _el(pos, kind) - if kind in QTY_ELEM and p.get("kindQty") is not None: - _el(ke, QTY_ELEM[kind], f'{p["kindQty"]:.2f}') - - if data.get("shareClasses"): - sce = _el(_el(fund, "SingleFund"), "ShareClasses") - for sc in data["shareClasses"]: - x = _el(sce, "ShareClass") - _el(_el(x, "Identifiers"), "ISIN", sc["isin"]) - if sc.get("officialName"): - _el(_el(x, "Names"), "OfficialName", sc["officialName"]) - _el(x, "Currency", sc["currency"]) - if sc.get("navPrice") is not None: - pr = _el(_el(x, "Prices"), "Price") - _el(pr, "ActionCode", "C") - _el(pr, "NavDate", fu["navDate"]) - _el(pr, "PriceCurrency", sc["currency"]) - _el(pr, "PriceNature", "OFFICIAL") - _el(pr, "NavPrice", f'{sc["navPrice"]:.2f}') - if sc.get("navFundCcy") is not None: - t = _el(_el(x, "TotalAssetValues"), "TotalAssetValue") - _el(t, "NavDate", fu["navDate"]) - _el(t, "TotalAssetNature", "OFFICIAL") - _el(_el(t, "TotalNetAssetValue"), "Amount", - f'{sc["navFundCcy"]:.2f}', ccy=ccy) - if sc.get("sharesOutstanding") is not None: - _el(t, "SharesOutstanding", - f'{sc["sharesOutstanding"]:.0f}') + _el(ds, "SystemCountry", s["country"]) + _el(ds, "Short", s["short"]) + _el(ds, "Name", s["name"]) + _el(ds, "Type", s["type"]) + _el(cd, "DataOperation", dd.get("dataOperation")) + + funds_el = _el(root, "Funds") + for fj in data["funds"]: + ccy = fj["currency"] + fund = _el(funds_el, "Fund") + if fj.get("lei"): + _el(_el(fund, "Identifiers"), "LEI", fj["lei"]) + _el(_el(fund, "Names"), "OfficialName", fj["officialName"]) + _el(fund, "Currency", ccy) + if fj.get("singleFundFlag"): + _el(fund, "SingleFundFlag", fj["singleFundFlag"]) + fdd = _el(fund, "FundDynamicData") + tav = _el(_el(_el(fdd, "TotalAssetValues"), "TotalAssetValue"), + "NavDate", fj["navDate"]).getparent() + _el(tav, "TotalAssetNature", "OFFICIAL") + _el(_el(tav, "TotalNetAssetValue"), "Amount", + f'{fj["totalNav"]:.2f}', ccy=ccy) + ports = _el(fdd, "Portfolios") + for pj in fj["portfolios"]: + pe = _el(ports, "Portfolio") + _el(pe, "NavDate", pj["navDate"]) + poss = _el(pe, "Positions") + for p in pj["positions"]: + pos = _el(poss, "Position") + _el(pos, "UniqueID", p["uniqueId"]) + if p.get("isin"): + _el(_el(pos, "Identifiers"), "ISIN", p["isin"]) + if p.get("currency"): + _el(pos, "Currency", p["currency"]) + _el(_el(pos, "TotalValue"), "Amount", + f'{p["value"]:.2f}', ccy=ccy) + _el(pos, "TotalPercentage", f'{p["percentage"]:.2f}') + kind = p["kind"] if p.get("kind") in POSITION_KINDS \ + else "Generic" + ke = _el(pos, kind) + if kind in QTY_ELEM and p.get("kindQty") is not None: + _el(ke, QTY_ELEM[kind], f'{p["kindQty"]:.2f}') + if fj.get("shareClasses"): + sce = _el(_el(fund, "SingleFund"), "ShareClasses") + for sc in fj["shareClasses"]: + x = _el(sce, "ShareClass") + _el(_el(x, "Identifiers"), "ISIN", sc["isin"]) + if sc.get("officialName"): + _el(_el(x, "Names"), "OfficialName", sc["officialName"]) + _el(x, "Currency", sc["currency"]) + if sc.get("navPrice") is not None: + pr = _el(_el(x, "Prices"), "Price") + _el(pr, "ActionCode", "C") + _el(pr, "NavDate", fj["navDate"]) + _el(pr, "PriceCurrency", sc["currency"]) + _el(pr, "PriceNature", "OFFICIAL") + _el(pr, "NavPrice", f'{sc["navPrice"]:.2f}') + if sc.get("navFundCcy") is not None: + t = _el(_el(x, "TotalAssetValues"), "TotalAssetValue") + _el(t, "NavDate", fj["navDate"]) + _el(t, "TotalAssetNature", "OFFICIAL") + _el(_el(t, "TotalNetAssetValue"), "Amount", + f'{sc["navFundCcy"]:.2f}', ccy=ccy) + if sc.get("sharesOutstanding") is not None: + _el(t, "SharesOutstanding", + f'{sc["sharesOutstanding"]:.0f}') if data.get("assets"): amd = _el(root, "AssetMasterData") @@ -203,7 +227,7 @@ def from_json(data: dict, out_path: str) -> None: _el(ae, "UniqueID", a["uniqueId"]) if a.get("isin"): _el(_el(ae, "Identifiers"), "ISIN", a["isin"]) - _el(ae, "Currency", a.get("currency") or ccy) + _el(ae, "Currency", a.get("currency")) if a.get("country"): _el(ae, "Country", a["country"]) _el(ae, "Name", a["name"]) diff --git a/Database_Integration/README.md b/Database_Integration/README.md index b30f2fd..1a8be84 100644 --- a/Database_Integration/README.md +++ b/Database_Integration/README.md @@ -1,48 +1,94 @@ # Database Integration -![python](https://img.shields.io/badge/SQLite%20round--trip-verified-brightgreen) ![sql](https://img.shields.io/badge/Oracle%2FSQLServer%2FPostgres-code--reference-blue) +![python](https://img.shields.io/badge/Python-verified-brightgreen) ![java](https://img.shields.io/badge/Java-verified-brightgreen) ![node](https://img.shields.io/badge/JavaScript-verified-brightgreen) ![csharp](https://img.shields.io/badge/C%23-verified-brightgreen) ![multi-fund](https://img.shields.io/badge/multi--node-yes-blue) -FundsXML ⇄ relational database, both directions. Per project scope **no -database is provisioned** (no Docker): the Oracle / SQL Server / PostgreSQL -files are **code references**, and the **SQLite Python implementation is the -runnable, verified reference** (stdlib + lxml only) that exercises the full -round-trip and produces XSD-valid FundsXML. +Standalone, copy-me examples that move data **both directions** between FundsXML +and a relational database, in the language you already use: -## Layout +| Language | File | DB driver | Verified | +|----------|------|-----------|----------| +| Python | [`python/fundsxml_db.py`](python/fundsxml_db.py) | stdlib `sqlite3` | ✅ locally | +| Java | [`java/FundsXmlDb.java`](java/FundsXmlDb.java) | `sqlite-jdbc` (native `javax.xml`, no JAXB) | ✅ locally | +| JavaScript | [`javascript/fundsxml_db.mjs`](javascript/fundsxml_db.mjs) | `sql.js` (pure-WASM) | ✅ locally | +| C# / .NET | [`csharp/FundsXmlDb.cs`](csharp/FundsXmlDb.cs) | `Microsoft.Data.Sqlite` | ✅ locally + CI | -| Path | What | -|------|------| -| [`ddl/schema.sql`](ddl/schema.sql) | Shared relational model (fund, share_class, asset, position) | -| [`load_from_fundsxml/`](load_from_fundsxml/) | FundsXML → DB: `postgres.sql` (XMLTABLE), `oracle.sql` (XMLType + XMLTABLE), `sqlserver.sql` (`.nodes()`/`.value()`) | -| [`generate_fundsxml/`](generate_fundsxml/) | DB → FundsXML: SQL/XML publishing (`xmlelement`/`xmlagg`, `XMLElement`/`XMLAgg`, `FOR XML PATH`) | -| [`python/fundsxml_db.py`](python/fundsxml_db.py) | **Runnable** SQLite reference: `init` / `load` / `generate` / `roundtrip` | +Every program is **self-contained**, heavily commented as a teaching artifact +(read one top-to-bottom and reimplement the pattern), and exposes the **same +CLI**: -The relational model is keyed by `ControlData/UniqueDocumentID` so multiple -documents coexist. FundsXML 4.x has no XML namespace — all XPath/queries use -bare element names. +``` + init create the schema + import FundsXML -> rows + export rows -> FundsXML + roundtrip import then export through the DB +``` + +## Relational model — multi-node + +[`ddl/schema.sql`](ddl/schema.sql): `document → fund → portfolio → position`, +`share_class` per fund, document-scoped `asset`. 1-based ordinal keys +(`fund_seq`, `portfolio_seq`, `position_seq`) capture **multiple funds, multiple +portfolios, multiple positions** and preserve their order, so the export is +deterministic and faithful. SQLite makes the examples zero-setup; the same SQL +runs on PostgreSQL (Oracle/SQL Server need only type tweaks — noted in the DDL). + +## Round-trip is proven by equivalence + +The user requirement: export the data the import wrote, then check the import +and export files are the same. [`tools/xml_equiv.py`](tools/xml_equiv.py) is the +shared comparator — two FundsXML files are equal when they differ only by the +volatile `ControlData/DocumentGenerated` timestamp, numeric formatting, +whitespace, and attribute/namespace ordering; **everything else, including +child order, is significant**. It is always paired with XSD validation +(`xml_equiv` deliberately ignores `xmlns`, so XSD is the complementary check). -## Runnable round-trip (verified) +The fixture +[`FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml`](../FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml) +(2 funds; one with 2 portfolios; share classes; document-scoped assets) is +authored **round-trip-faithful** — it contains only what the model captures, so +all four languages reproduce it **exactly** (verified: each export is +`xml_equiv`-equal to the original *and* to every other language's export, and +XSD-valid). The canonical `Mixed-Fund_Positions.xml` also round-trips (n=1) +XSD-valid; it is intentionally lossy outside the positions core, so only its +XSD-validity is asserted. + +## Run ```bash -python3 Database_Integration/python/fundsxml_db.py roundtrip \ - FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml regenerated.xml +tools/fetch-schema.sh 4.2.9 # XSD for validation +FX=FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml + +# Python +python3 Database_Integration/python/fundsxml_db.py roundtrip "$FX" out.xml + +# Java (sqlite-jdbc fetched into .lib/ by tools/fetch-tools.sh) +tools/fetch-tools.sh +CP=.lib/sqlite-jdbc-3.46.1.3.jar +javac -cp "$CP" -d /tmp/db Database_Integration/java/FundsXmlDb.java +java --enable-native-access=ALL-UNNAMED -cp "$CP:/tmp/db" FundsXmlDb roundtrip "$FX" out.xml + +# JavaScript +( cd Database_Integration/javascript && npm install ) +node Database_Integration/javascript/fundsxml_db.mjs roundtrip "$FX" out.xml + +# C# +dotnet run --project Database_Integration/csharp -- roundtrip "$FX" out.xml -tools/fetch-schema.sh 4.2.9 -xmllint --noout --schema .schema-cache/4.2.9/FundsXML.xsd regenerated.xml +# prove it: identical to the input, and schema-valid +python3 Database_Integration/tools/xml_equiv.py "$FX" out.xml +xmllint --noout --schema .schema-cache/4.2.9/FundsXML.xsd out.xml ``` -Verified for the 4.2.9 positions, 4.1.0 and 4.0.0 positions, and 4.2.9 -transactions samples: each loads into SQLite and regenerates **XSD-valid** -FundsXML with NAV, position count and percentage sum preserved. +(`--enable-native-access=ALL-UNNAMED` only silences a JDK 24+ warning when +sqlite-jdbc loads its native library; the example works without it.) -### Round-trip fidelity +## Engine-specific SQL references (not executed) -The model captures fund header, share classes (incl. `TotalAssetValues` / -`SharesOutstanding`), positions (value, %, class + quantity child) and asset -master data. It is intentionally **lossy** for everything outside that core -(issuer details, derivative terms, transactions, regulatory blocks) — the -regenerated file is a faithful *positions* document, normalized to the 4.2.9 -schema (a 4.0.0 input comes back without `ControlData/Version`, which is valid). +[`load_from_fundsxml/`](load_from_fundsxml/) and +[`generate_fundsxml/`](generate_fundsxml/) show the **multi-fund** shred/publish +pattern in Oracle, SQL Server and PostgreSQL SQL (nested `XMLTABLE` / +`.nodes()` with ordinality; grouped `XMLAGG` / `FOR XML PATH`). They are code +references mirroring the runnable examples; no database is provisioned here. -The three enterprise-DB SQL files mirror the same shred/publish logic; only the -SQLite path is executed here. CI runs the round-trip on every push. +CI runs the round-trip for all four languages on the multi-fund fixture +(equivalence + XSD), plus the single-fund sample, on every push. diff --git a/Database_Integration/csharp/FundsXmlDb.cs b/Database_Integration/csharp/FundsXmlDb.cs new file mode 100644 index 0000000..9a8f5d7 --- /dev/null +++ b/Database_Integration/csharp/FundsXmlDb.cs @@ -0,0 +1,506 @@ +// ============================================================================= +// FundsXML <-> relational database — runnable reference (C# / .NET, SQLite). +// +// PURPOSE +// Standalone, copy-me example of BOTH directions (import a FundsXML file into +// the relational schema, export it back). Over-commented on purpose so it +// doubles as documentation for your own implementation. +// +// DB SCHEMA ../ddl/schema.sql (document -> fund -> portfolio -> position; +// share_class per fund; asset document-scoped). +// +// RUN +// dotnet run --project Database_Integration/csharp -- \ +// roundtrip FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml out.xml +// commands: init | import | export | +// roundtrip +// +// DEPENDENCIES +// Microsoft.Data.Sqlite (bundles native SQLite) + System.Xml in the BCL. +// No XSD data-binding: System.Xml DOM/XPath keeps it small and +// version-tolerant (FundsXML 4.x is backward compatible). +// +// FUNDSXML ASSUMPTIONS +// * No XML namespace -> bare element names in XPath. +// * Many //: all iterated; 1-based *_seq columns +// preserve order so the round-trip compares equal (../tools/xml_equiv.py). +// * Positions link to AssetMasterData by shared -> `asset` is +// document-scoped. +// * Export normalized to the 4.2.9 schema URL; constants the model does not +// store (TotalAssetNature=OFFICIAL, Price ActionCode=C / PriceNature= +// OFFICIAL) are reproduced verbatim. +// +// SECURITY +// XmlReaderSettings: DtdProcessing.Prohibit + XmlResolver=null (no XXE). +// ============================================================================= +using System; +using System.Globalization; +using System.IO; +using System.Xml; +using Microsoft.Data.Sqlite; + +internal static class FundsXmlDb +{ + const string SchemaUrl = + "https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd"; + + // Position instrument-class element + its mandatory quantity child. + static readonly System.Collections.Generic.HashSet PositionKinds = + new() { "Equity", "Bond", "ShareClass", "Warrant", "Certificate", + "Option", "Future", "FXForward", "Swap", "Repo", "RealEstate", + "CallMoney", "Account", "Generic" }; + static readonly System.Collections.Generic.Dictionary QtyElem = + new() { ["Equity"] = "Units", ["Warrant"] = "Units", + ["Certificate"] = "Units", ["Bond"] = "Nominal", + ["ShareClass"] = "Shares", ["Option"] = "Contracts", + ["Future"] = "Contracts" }; + + static string Inv(double v) => v.ToString("0.00", CultureInfo.InvariantCulture); + + // ---- helpers ----------------------------------------------------------- + static string? T(XmlNode ctx, string xpath) + { + var n = ctx.SelectSingleNode(xpath); + return string.IsNullOrEmpty(n?.InnerText) ? null : n!.InnerText; + } + + static object DbNum(string? s) => + string.IsNullOrWhiteSpace(s) ? DBNull.Value + : double.Parse(s, CultureInfo.InvariantCulture); + + static XmlElement El(XmlDocument doc, XmlNode parent, string tag, + string? text = null) + { + var e = doc.CreateElement(tag); + if (text != null) e.AppendChild(doc.CreateTextNode(text)); + parent.AppendChild(e); + return e; + } + + static SqliteConnection Open(string db) + { + var c = new SqliteConnection($"Data Source={db}"); + c.Open(); + using var pragma = c.CreateCommand(); + pragma.CommandText = "PRAGMA foreign_keys = ON"; + pragma.ExecuteNonQuery(); + return c; + } + + static void Exec(SqliteConnection c, string sql) + { + using var cmd = c.CreateCommand(); + cmd.CommandText = sql; + cmd.ExecuteNonQuery(); + } + + // Bind helper: positional @p0,@p1,... mirrors the other languages' "?". + static void Run(SqliteConnection c, string sql, params object?[] ps) + { + using var cmd = c.CreateCommand(); + cmd.CommandText = sql; + for (int i = 0; i < ps.Length; i++) + cmd.Parameters.AddWithValue($"@p{i}", ps[i] ?? DBNull.Value); + cmd.ExecuteNonQuery(); + } + + static XmlDocument ParseSecure(string xml) + { + var settings = new XmlReaderSettings + { + DtdProcessing = DtdProcessing.Prohibit, // reject DOCTYPE (XXE) + XmlResolver = null // no external entities + }; + var doc = new XmlDocument(); + using var r = XmlReader.Create(xml, settings); + doc.Load(r); + return doc; + } + + static string DdlPath() + { + // Works whether run from repo root or the csharp/ project dir. + foreach (var p in new[] { Path.Combine("Database_Integration", "ddl", + "schema.sql"), Path.Combine("..", "ddl", "schema.sql") }) + if (File.Exists(p)) return p; + throw new FileNotFoundException("ddl/schema.sql not found"); + } + + static void Init(string db) + { + // Strip "--" line comments (no string literals in the DDL) then run + // each ";"-separated statement. + var lines = File.ReadAllLines(DdlPath()); + var sb = new System.Text.StringBuilder(); + foreach (var line in lines) + { + int cut = line.IndexOf("--", StringComparison.Ordinal); + sb.AppendLine(cut >= 0 ? line[..cut] : line); + } + using var c = Open(db); + foreach (var stmt in sb.ToString().Split(';')) + if (stmt.Trim().Length > 0) Exec(c, stmt); + } + + // ---- import : FundsXML -> rows ----------------------------------------- + static string DoImport(string db, string xml) + { + var doc = ParseSecure(xml); + var cd = doc.SelectSingleNode("/FundsXML4/ControlData")!; + var docId = T(cd, "UniqueDocumentID")!; + + using var c = Open(db); + using var tx = c.BeginTransaction(); + + Run(c, "INSERT INTO document VALUES (@p0,@p1,@p2,@p3,@p4,@p5,@p6,@p7,@p8)", + docId, T(cd, "DocumentGenerated"), T(cd, "Version"), + T(cd, "ContentDate"), T(cd, "DataOperation"), + T(cd, "DataSupplier/SystemCountry"), T(cd, "DataSupplier/Short"), + T(cd, "DataSupplier/Name"), T(cd, "DataSupplier/Type")); + + var funds = doc.SelectNodes("/FundsXML4/Funds/Fund")!; + for (int fi = 0; fi < funds.Count; fi++) + { + var fund = funds[fi]!; + int fundSeq = fi + 1; // 1-based doc order + var ccy = T(fund, "Currency")!; + var tav = fund.SelectSingleNode( + "FundDynamicData/TotalAssetValues/TotalAssetValue")!; + Run(c, "INSERT INTO fund VALUES (@p0,@p1,@p2,@p3,@p4,@p5,@p6,@p7)", + docId, fundSeq, T(fund, "Identifiers/LEI"), + T(fund, "Names/OfficialName"), ccy, + T(fund, "SingleFundFlag"), T(tav, "NavDate"), + double.Parse(T(tav, + $"TotalNetAssetValue/Amount[@ccy='{ccy}']")!, + CultureInfo.InvariantCulture)); + + var scs = fund.SelectNodes("SingleFund/ShareClasses/ShareClass")!; + foreach (XmlNode sc in scs) + Run(c, "INSERT INTO share_class VALUES (@p0,@p1,@p2,@p3,@p4,@p5,@p6,@p7)", + docId, fundSeq, T(sc, "Identifiers/ISIN"), + T(sc, "Names/OfficialName"), T(sc, "Currency"), + DbNum(T(sc, "Prices/Price/NavPrice")), + DbNum(T(sc, $"TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount[@ccy='{ccy}']")), + DbNum(T(sc, "TotalAssetValues/TotalAssetValue/SharesOutstanding"))); + + var ports = fund.SelectNodes( + "FundDynamicData/Portfolios/Portfolio")!; + for (int pi = 0; pi < ports.Count; pi++) + { + var port = ports[pi]!; + int portSeq = pi + 1; + Run(c, "INSERT INTO portfolio VALUES (@p0,@p1,@p2,@p3)", + docId, fundSeq, portSeq, T(port, "NavDate")); + var poss = port.SelectNodes("Positions/Position")!; + for (int qi = 0; qi < poss.Count; qi++) + { + var pos = poss[qi]!; + string? kind = null; + foreach (XmlNode ch in pos.ChildNodes) + if (ch.NodeType == XmlNodeType.Element + && PositionKinds.Contains(ch.Name)) + { kind = ch.Name; break; } + object qty = (kind != null && QtyElem.ContainsKey(kind)) + ? DbNum(T(pos, $"{kind}/{QtyElem[kind]}")) + : DBNull.Value; + Run(c, "INSERT INTO position VALUES (@p0,@p1,@p2,@p3,@p4,@p5,@p6,@p7,@p8,@p9,@p10)", + docId, fundSeq, portSeq, qi + 1, + T(pos, "UniqueID"), T(pos, "Identifiers/ISIN"), + T(pos, "Currency"), + double.Parse(T(pos, $"TotalValue/Amount[@ccy='{ccy}']")!, + CultureInfo.InvariantCulture), + double.Parse(T(pos, "TotalPercentage")!, + CultureInfo.InvariantCulture), + kind, qty); + } + } + } + + var assets = doc.SelectNodes("/FundsXML4/AssetMasterData/Asset")!; + foreach (XmlNode a in assets) + Run(c, "INSERT INTO asset VALUES (@p0,@p1,@p2,@p3,@p4,@p5,@p6)", + docId, T(a, "UniqueID"), T(a, "Identifiers/ISIN"), + T(a, "Name"), T(a, "AssetType"), T(a, "Currency"), + T(a, "Country")); + + tx.Commit(); + return docId; + } + + // ---- export : rows -> FundsXML ---------------------------------------- + static SqliteDataReader Q(SqliteConnection c, string sql, + params object?[] ps) + { + var cmd = c.CreateCommand(); + cmd.CommandText = sql; + for (int i = 0; i < ps.Length; i++) + cmd.Parameters.AddWithValue($"@p{i}", ps[i] ?? DBNull.Value); + return cmd.ExecuteReader(); + } + + static string? S(SqliteDataReader r, string col) => + r.IsDBNull(r.GetOrdinal(col)) ? null : r.GetString(r.GetOrdinal(col)); + static double D(SqliteDataReader r, string col) => + Convert.ToDouble(r.GetValue(r.GetOrdinal(col)), + CultureInfo.InvariantCulture); + static bool Null(SqliteDataReader r, string col) => + r.IsDBNull(r.GetOrdinal(col)); + + static void Export(string db, string docId, string outPath) + { + using var c = Open(db); + var doc = new XmlDocument(); + var root = doc.CreateElement("FundsXML4"); + // xsi:noNamespaceSchemaLocation must really live in the + // XMLSchema-instance namespace, otherwise a validator rejects it + // ("attribute 'noNamespaceSchemaLocation' is not allowed"). Create it + // as a properly namespaced attribute (this also emits xmlns:xsi). + const string XsiNs = "http://www.w3.org/2001/XMLSchema-instance"; + var sl = doc.CreateAttribute("xsi", "noNamespaceSchemaLocation", XsiNs); + sl.Value = SchemaUrl; + root.Attributes.Append(sl); + doc.AppendChild(root); + + using (var d = Q(c, "SELECT * FROM document WHERE document_id=@p0", + docId)) + { + if (!d.Read()) throw new Exception($"no document {docId}"); + var cd = El(doc, root, "ControlData"); + El(doc, cd, "UniqueDocumentID", S(d, "document_id")); + // Regenerate timestamp; xml_equiv.py ignores its value. + El(doc, cd, "DocumentGenerated", + S(d, "generated") ?? "2025-10-02T00:00:00"); + if (!Null(d, "version")) // none for 4.0.0 + El(doc, cd, "Version", S(d, "version")); + El(doc, cd, "ContentDate", S(d, "content_date")); + var ds = El(doc, cd, "DataSupplier"); + El(doc, ds, "SystemCountry", S(d, "supplier_country")); + El(doc, ds, "Short", S(d, "supplier_short")); + El(doc, ds, "Name", S(d, "supplier_name")); + El(doc, ds, "Type", S(d, "supplier_type")); + El(doc, cd, "DataOperation", S(d, "data_operation")); + } + + var fundsEl = El(doc, root, "Funds"); + var fundRows = new System.Collections.Generic.List< + System.Collections.Generic.Dictionary>(); + using (var fr = Q(c, "SELECT * FROM fund WHERE document_id=@p0 " + + "ORDER BY fund_seq", docId)) + while (fr.Read()) + fundRows.Add(new() { + ["fund_seq"] = fr.GetInt32(fr.GetOrdinal("fund_seq")), + ["currency"] = S(fr, "currency"), + ["lei"] = S(fr, "lei"), + ["official_name"] = S(fr, "official_name"), + ["single_fund_flag"] = S(fr, "single_fund_flag"), + ["nav_date"] = S(fr, "nav_date"), + ["total_nav"] = D(fr, "total_nav") }); + + foreach (var f in fundRows) + { + var ccy = (string)f["currency"]!; + int fundSeq = (int)f["fund_seq"]!; + var navDate = (string?)f["nav_date"]; + var fund = El(doc, fundsEl, "Fund"); + if (f["lei"] != null) + El(doc, El(doc, fund, "Identifiers"), "LEI", + (string?)f["lei"]); + El(doc, El(doc, fund, "Names"), "OfficialName", + (string?)f["official_name"]); + El(doc, fund, "Currency", ccy); + if (f["single_fund_flag"] != null) + El(doc, fund, "SingleFundFlag", + (string?)f["single_fund_flag"]); + + var fdd = El(doc, fund, "FundDynamicData"); + var tavp = El(doc, El(doc, fdd, "TotalAssetValues"), + "TotalAssetValue"); + El(doc, tavp, "NavDate", navDate); + El(doc, tavp, "TotalAssetNature", "OFFICIAL"); + var amt = El(doc, El(doc, tavp, "TotalNetAssetValue"), "Amount", + Inv((double)f["total_nav"]!)); + amt.SetAttribute("ccy", ccy); + + var ports = El(doc, fdd, "Portfolios"); + var portRows = new System.Collections.Generic.List<(int seq, + string? nav)>(); + using (var pr = Q(c, "SELECT portfolio_seq,nav_date FROM portfolio" + + " WHERE document_id=@p0 AND fund_seq=@p1 ORDER BY " + + "portfolio_seq", docId, fundSeq)) + while (pr.Read()) + portRows.Add((pr.GetInt32(0), + pr.IsDBNull(1) ? null : pr.GetString(1))); + foreach (var (pseq, pnav) in portRows) + { + var pe = El(doc, ports, "Portfolio"); + El(doc, pe, "NavDate", pnav); + var poss = El(doc, pe, "Positions"); + var posRows = new System.Collections.Generic.List< + System.Collections.Generic.Dictionary>(); + using (var qr = Q(c, "SELECT * FROM position WHERE " + + "document_id=@p0 AND fund_seq=@p1 AND " + + "portfolio_seq=@p2 ORDER BY position_seq", + docId, fundSeq, pseq)) + while (qr.Read()) + posRows.Add(new() { + ["unique_id"] = S(qr, "unique_id"), + ["isin"] = S(qr, "isin"), + ["currency"] = S(qr, "currency"), + ["value"] = D(qr, "value_fund_ccy"), + ["pct"] = D(qr, "percentage"), + ["kind"] = S(qr, "kind"), + ["qty"] = Null(qr, "kind_qty") + ? (object?)null : D(qr, "kind_qty") }); + foreach (var p in posRows) + { + var pos = El(doc, poss, "Position"); + El(doc, pos, "UniqueID", (string?)p["unique_id"]); + if (p["isin"] != null) + El(doc, El(doc, pos, "Identifiers"), "ISIN", + (string?)p["isin"]); + if (p["currency"] != null) + El(doc, pos, "Currency", (string?)p["currency"]); + var tv = El(doc, El(doc, pos, "TotalValue"), "Amount", + Inv((double)p["value"]!)); + tv.SetAttribute("ccy", ccy); + El(doc, pos, "TotalPercentage", Inv((double)p["pct"]!)); + var kind = p["kind"] is string k + && PositionKinds.Contains(k) ? k : "Generic"; + var ke = El(doc, pos, kind); + if (QtyElem.ContainsKey(kind) && p["qty"] != null) + El(doc, ke, QtyElem[kind], Inv((double)p["qty"]!)); + } + } + + var scRows = new System.Collections.Generic.List< + System.Collections.Generic.Dictionary>(); + using (var sr = Q(c, "SELECT * FROM share_class WHERE " + + "document_id=@p0 AND fund_seq=@p1 ORDER BY isin", + docId, fundSeq)) + while (sr.Read()) + scRows.Add(new() { + ["isin"] = S(sr, "isin"), + ["official_name"] = S(sr, "official_name"), + ["currency"] = S(sr, "currency"), + ["nav_price"] = Null(sr, "nav_price") + ? (object?)null : D(sr, "nav_price"), + ["nav_fund_ccy"] = Null(sr, "nav_fund_ccy") + ? (object?)null : D(sr, "nav_fund_ccy"), + ["shares"] = Null(sr, "shares_outstanding") + ? (object?)null : D(sr, "shares_outstanding") }); + if (scRows.Count > 0) + { + var sce = El(doc, El(doc, fund, "SingleFund"), + "ShareClasses"); + foreach (var sc in scRows) + { + var x = El(doc, sce, "ShareClass"); + El(doc, El(doc, x, "Identifiers"), "ISIN", + (string?)sc["isin"]); + if (sc["official_name"] != null) + El(doc, El(doc, x, "Names"), "OfficialName", + (string?)sc["official_name"]); + El(doc, x, "Currency", (string?)sc["currency"]); + if (sc["nav_price"] != null) + { + var pr2 = El(doc, El(doc, x, "Prices"), "Price"); + El(doc, pr2, "ActionCode", "C"); + El(doc, pr2, "NavDate", navDate); + El(doc, pr2, "PriceCurrency", + (string?)sc["currency"]); + El(doc, pr2, "PriceNature", "OFFICIAL"); + El(doc, pr2, "NavPrice", + Inv((double)sc["nav_price"]!)); + } + if (sc["nav_fund_ccy"] != null) + { + var t2 = El(doc, El(doc, x, "TotalAssetValues"), + "TotalAssetValue"); + El(doc, t2, "NavDate", navDate); + El(doc, t2, "TotalAssetNature", "OFFICIAL"); + var a2 = El(doc, El(doc, t2, "TotalNetAssetValue"), + "Amount", Inv((double)sc["nav_fund_ccy"]!)); + a2.SetAttribute("ccy", ccy); + if (sc["shares"] != null) + El(doc, t2, "SharesOutstanding", + ((double)sc["shares"]!).ToString("0", + CultureInfo.InvariantCulture)); + } + } + } + } + + var assetRows = new System.Collections.Generic.List< + System.Collections.Generic.Dictionary>(); + using (var ar = Q(c, "SELECT * FROM asset WHERE document_id=@p0 " + + "ORDER BY unique_id", docId)) + while (ar.Read()) + assetRows.Add(new() { + ["unique_id"] = S(ar, "unique_id"), + ["isin"] = S(ar, "isin"), + ["currency"] = S(ar, "currency"), + ["country"] = S(ar, "country"), + ["name"] = S(ar, "name"), + ["asset_type"] = S(ar, "asset_type") }); + if (assetRows.Count > 0) + { + var amd = El(doc, root, "AssetMasterData"); + foreach (var a in assetRows) + { + var ae = El(doc, amd, "Asset"); + El(doc, ae, "UniqueID", a["unique_id"]); + if (a["isin"] != null) + El(doc, El(doc, ae, "Identifiers"), "ISIN", a["isin"]); + El(doc, ae, "Currency", a["currency"]); + if (a["country"] != null) + El(doc, ae, "Country", a["country"]); + El(doc, ae, "Name", a["name"]); + El(doc, ae, "AssetType", a["asset_type"]); + } + } + + var settings = new XmlWriterSettings + { Indent = true, Encoding = new System.Text.UTF8Encoding(false) }; + using var w = XmlWriter.Create(outPath, settings); + doc.Save(w); + } + + static int Main(string[] args) + { + if (args.Length == 0) + { + Console.Error.WriteLine("usage: init|import|export|roundtrip ..."); + return 2; + } + switch (args[0]) + { + case "init": + Init(args[1]); + break; + case "import": + Console.WriteLine("imported " + DoImport(args[1], args[2])); + break; + case "export": + Export(args[1], args[2], args[3]); + Console.WriteLine("wrote " + args[3]); + break; + case "roundtrip": + { + // import then export THROUGH the DB (the required test). + var tmp = Path.GetTempFileName(); + File.Delete(tmp); + Init(tmp); + var id = DoImport(tmp, args[1]); + Export(tmp, id, args[2]); + File.Delete(tmp); + Console.WriteLine($"round-trip ok: {args[1]} -> DB -> " + + $"{args[2]} (doc {id})"); + break; + } + default: + Console.Error.WriteLine("unknown: " + args[0]); + return 2; + } + return 0; + } +} diff --git a/Database_Integration/csharp/FundsXmlDb.csproj b/Database_Integration/csharp/FundsXmlDb.csproj new file mode 100644 index 0000000..63a3ac2 --- /dev/null +++ b/Database_Integration/csharp/FundsXmlDb.csproj @@ -0,0 +1,16 @@ + + + + Exe + net8.0 + enable + disable + FundsXmlDb + FundsXml.DbExample + + + + + diff --git a/Database_Integration/ddl/schema.sql b/Database_Integration/ddl/schema.sql index 29e0d83..e8a8707 100644 --- a/Database_Integration/ddl/schema.sql +++ b/Database_Integration/ddl/schema.sql @@ -1,65 +1,108 @@ --- Shared relational model for FundsXML positions data. +-- =========================================================================== +-- Relational model for FundsXML positions data — MULTI-FUND capable. -- --- Portable ANSI-ish DDL (works on SQLite/Postgres; see notes for Oracle and --- SQL Server type tweaks). One row set per source document, keyed by the --- ControlData/UniqueDocumentID so multiple files can coexist. +-- A FundsXML document is a tree: +-- FundsXML4 +-- ControlData -> table `document` (1 per file) +-- Funds/Fund (1..n) -> table `fund` +-- SingleFund/ShareClasses/ShareClass -> table `share_class` +-- FundDynamicData/Portfolios/Portfolio (1..n) -> table `portfolio` +-- Positions/Position (1..n) -> table `position` +-- AssetMasterData/Asset (1..n) -> table `asset` (document-scoped) -- --- Faithful enough to round-trip the canonical positions sample back to --- XSD-valid FundsXML (fund + share classes + positions + asset master data). +-- WHY surrogate ordinal keys (*_seq): +-- FundsXML does not guarantee a stable business key for a Fund or Portfolio +-- (LEI is optional and not always unique within a file; a Portfolio has no +-- id at all). To round-trip *exactly* — same number of funds/portfolios, +-- same order, duplicates preserved — we number them by their position in +-- the document (fund_seq, portfolio_seq, position_seq, 1-based). This makes +-- the export deterministic and the import lossless for the captured model. +-- +-- WHY `asset` is document-scoped (no fund_seq): +-- AssetMasterData sits directly under FundsXML4, shared by all funds, and +-- Asset/UniqueID is an xs:ID (unique per document). Positions reference it +-- by that UniqueID — exactly the FundsXML Position<->Asset link. +-- +-- Portable ANSI-ish DDL: runs as-is on SQLite (used by the runnable examples) +-- and PostgreSQL. For the other engines adjust types: +-- Oracle : VARCHAR(n)->VARCHAR2(n), DECIMAL->NUMBER, CHAR ok +-- SQL Server : VARCHAR/DECIMAL ok; CHAR ok +-- (An XML staging column is shown in ../load_from_fundsxml/ for those engines.) +-- =========================================================================== -CREATE TABLE fund ( - document_id VARCHAR(128) NOT NULL, - lei VARCHAR(20), - official_name VARCHAR(256) NOT NULL, - currency CHAR(3) NOT NULL, - content_date DATE, - nav_date DATE, - total_nav DECIMAL(20,2) NOT NULL, - fxml_version VARCHAR(16), -- absent for 4.0.0 +CREATE TABLE document ( + document_id VARCHAR(128) NOT NULL, + generated VARCHAR(32), -- ControlData/DocumentGenerated + version VARCHAR(16), -- absent for FundsXML 4.0.0 + content_date DATE, + data_operation VARCHAR(16), -- INITIAL / DELTA / ... + supplier_country CHAR(2), + supplier_short VARCHAR(64), + supplier_name VARCHAR(256), + supplier_type VARCHAR(64), PRIMARY KEY (document_id) ); +CREATE TABLE fund ( + document_id VARCHAR(128) NOT NULL, + fund_seq INTEGER NOT NULL, -- 1-based order within Funds + lei VARCHAR(20), + official_name VARCHAR(256) NOT NULL, + currency CHAR(3) NOT NULL, + single_fund_flag VARCHAR(8), + nav_date DATE, + total_nav DECIMAL(20,2) NOT NULL, + PRIMARY KEY (document_id, fund_seq), + FOREIGN KEY (document_id) REFERENCES document (document_id) +); + CREATE TABLE share_class ( - document_id VARCHAR(128) NOT NULL, - isin CHAR(12) NOT NULL, - official_name VARCHAR(256), - currency CHAR(3) NOT NULL, - nav_price DECIMAL(20,6), - nav_fund_ccy DECIMAL(20,2), -- NAV in fund currency + document_id VARCHAR(128) NOT NULL, + fund_seq INTEGER NOT NULL, + isin CHAR(12) NOT NULL, + official_name VARCHAR(256), + currency CHAR(3) NOT NULL, + nav_price DECIMAL(20,6), + nav_fund_ccy DECIMAL(20,2), -- NAV in the fund currency shares_outstanding DECIMAL(28,6), - PRIMARY KEY (document_id, isin), - FOREIGN KEY (document_id) REFERENCES fund (document_id) + PRIMARY KEY (document_id, fund_seq, isin), + FOREIGN KEY (document_id, fund_seq) REFERENCES fund (document_id, fund_seq) ); -CREATE TABLE asset ( - document_id VARCHAR(128) NOT NULL, - unique_id VARCHAR(256) NOT NULL, - isin CHAR(12), - name VARCHAR(256), - asset_type CHAR(2), -- EQ, BO, SC, ... - currency CHAR(3), - country CHAR(2), - PRIMARY KEY (document_id, unique_id), - FOREIGN KEY (document_id) REFERENCES fund (document_id) +CREATE TABLE portfolio ( + document_id VARCHAR(128) NOT NULL, + fund_seq INTEGER NOT NULL, + portfolio_seq INTEGER NOT NULL, -- 1-based within the fund + nav_date DATE, + PRIMARY KEY (document_id, fund_seq, portfolio_seq), + FOREIGN KEY (document_id, fund_seq) REFERENCES fund (document_id, fund_seq) ); CREATE TABLE position ( - document_id VARCHAR(128) NOT NULL, - unique_id VARCHAR(256) NOT NULL, -- joins to asset.unique_id - isin CHAR(12), - currency CHAR(3), - value_fund_ccy DECIMAL(20,2) NOT NULL, - percentage DECIMAL(9,4) NOT NULL, - kind VARCHAR(16), -- Position class element: - -- Equity/Bond/ShareClass/... - kind_qty DECIMAL(28,6), -- its quantity child - -- (Units/Nominal/Shares/Contracts) - PRIMARY KEY (document_id, unique_id), - FOREIGN KEY (document_id) REFERENCES fund (document_id) + document_id VARCHAR(128) NOT NULL, + fund_seq INTEGER NOT NULL, + portfolio_seq INTEGER NOT NULL, + position_seq INTEGER NOT NULL, -- 1-based within the portfolio + unique_id VARCHAR(256), -- joins to asset.unique_id + isin CHAR(12), + currency CHAR(3), + value_fund_ccy DECIMAL(20,2) NOT NULL, -- TotalValue in fund ccy + percentage DECIMAL(9,4) NOT NULL, -- TotalPercentage + kind VARCHAR(16), -- Equity/Bond/ShareClass/... + kind_qty DECIMAL(28,6), -- Units/Nominal/Shares/Contracts + PRIMARY KEY (document_id, fund_seq, portfolio_seq, position_seq), + FOREIGN KEY (document_id, fund_seq, portfolio_seq) + REFERENCES portfolio (document_id, fund_seq, portfolio_seq) ); --- Dialect notes --- Oracle : VARCHAR(n) -> VARCHAR2(n); DECIMAL -> NUMBER; CHAR ok. --- An XMLType staging column is shown in load_from_fundsxml. --- SQL Server : VARCHAR ok; DECIMAL ok; use an XML column for staging. --- Postgres : as-is; a native `xml` staging column is used for xmltable. +CREATE TABLE asset ( + document_id VARCHAR(128) NOT NULL, + unique_id VARCHAR(256) NOT NULL, + isin CHAR(12), + name VARCHAR(256), + asset_type CHAR(2), -- EQ, BO, SC, ... + currency CHAR(3), + country CHAR(2), + PRIMARY KEY (document_id, unique_id), + FOREIGN KEY (document_id) REFERENCES document (document_id) +); diff --git a/Database_Integration/generate_fundsxml/oracle.sql b/Database_Integration/generate_fundsxml/oracle.sql index c347617..93ec894 100644 --- a/Database_Integration/generate_fundsxml/oracle.sql +++ b/Database_Integration/generate_fundsxml/oracle.sql @@ -1,60 +1,76 @@ --- Oracle -> FundsXML (code reference; no DB is provisioned in this repo). +-- Oracle -> FundsXML, MULTI-FUND (code reference; no DB is provisioned). -- --- SQL/XML publishing: XMLElement / XMLAgg / XMLAttributes. Same FundsXML 4.2.9 --- positions shape as the Python reference. Bind :doc = fund.document_id. +-- SQL/XML publishing with XMLAGG. Multi-node essence: XMLAGG over the fund +-- rows (ORDER BY fund_seq) yields many ; nested correlated XMLAGG over +-- portfolio then position. Same canonical shape as the runnable examples +-- (xml_equiv.py-equal). :doc = document.document_id. Full per-column mapping: +-- ../ddl/schema.sql and the Python/Java/JavaScript/C# programs. -SELECT XMLElement("FundsXML4", - XMLAttributes( - 'http://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd' +SELECT XMLELEMENT("FundsXML4", + XMLATTRIBUTES( + 'http://www.w3.org/2001/XMLSchema-instance' AS "xmlns:xsi", + 'https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd' AS "xsi:noNamespaceSchemaLocation"), - XMLElement("ControlData", - XMLElement("UniqueDocumentID", f.document_id), - XMLElement("DocumentGenerated", '2025-10-02T00:00:00'), - CASE WHEN f.fxml_version IS NOT NULL - THEN XMLElement("Version", f.fxml_version) END, - XMLElement("ContentDate", TO_CHAR(f.content_date,'YYYY-MM-DD')), - XMLElement("DataSupplier", - XMLElement("SystemCountry",'AT'), - XMLElement("Short",'EURAM'), - XMLElement("Name",'Erste Asset Management GmbH'), - XMLElement("Type",'Asset Manager')), - XMLElement("DataOperation",'INITIAL')), - XMLElement("Funds", - XMLElement("Fund", - XMLElement("Identifiers", XMLElement("LEI", f.lei)), - XMLElement("Names", XMLElement("OfficialName", f.official_name)), - XMLElement("Currency", f.currency), - XMLElement("SingleFundFlag",'true'), - XMLElement("FundDynamicData", - XMLElement("TotalAssetValues", - XMLElement("TotalAssetValue", - XMLElement("NavDate", TO_CHAR(f.nav_date,'YYYY-MM-DD')), - XMLElement("TotalAssetNature",'OFFICIAL'), - XMLElement("TotalNetAssetValue", - XMLElement("Amount", - XMLAttributes(f.currency AS "ccy"), - TO_CHAR(f.total_nav,'FM999999999990.00'))))), - XMLElement("Portfolios", - XMLElement("Portfolio", - XMLElement("NavDate", TO_CHAR(f.nav_date,'YYYY-MM-DD')), - XMLElement("Positions", - (SELECT XMLAgg( - XMLElement("Position", - XMLElement("UniqueID", p.unique_id), - XMLElement("Currency", p.currency), - XMLElement("TotalValue", - XMLElement("Amount", - XMLAttributes(f.currency AS "ccy"), - TO_CHAR(p.value_fund_ccy,'FM999999999990.00'))), - XMLElement("TotalPercentage", - TO_CHAR(p.percentage,'FM990.00')), - XMLElement(EVALNAME p.kind)) - ORDER BY p.unique_id) - FROM position p - WHERE p.document_id = f.document_id))))))) - ).getClobVal() -FROM fund f -WHERE f.document_id = :doc; + (SELECT XMLELEMENT("ControlData", + XMLFOREST(d.document_id AS "UniqueDocumentID", + d.generated AS "DocumentGenerated", + d.version AS "Version", + TO_CHAR(d.content_date,'YYYY-MM-DD') AS "ContentDate"), + XMLELEMENT("DataSupplier", + XMLFOREST(d.supplier_country AS "SystemCountry", + d.supplier_short AS "Short", + d.supplier_name AS "Name", + d.supplier_type AS "Type")), + XMLELEMENT("DataOperation", d.data_operation)) + FROM document d WHERE d.document_id = :doc), + XMLELEMENT("Funds", + (SELECT XMLAGG( + XMLELEMENT("Fund", + XMLELEMENT("Identifiers", XMLELEMENT("LEI", f.lei)), + XMLELEMENT("Names", + XMLELEMENT("OfficialName", f.official_name)), + XMLELEMENT("Currency", f.currency), + XMLELEMENT("SingleFundFlag", f.single_fund_flag), + XMLELEMENT("FundDynamicData", + XMLELEMENT("TotalAssetValues", + XMLELEMENT("TotalAssetValue", + XMLFOREST(TO_CHAR(f.nav_date,'YYYY-MM-DD') AS "NavDate", + 'OFFICIAL' AS "TotalAssetNature"), + XMLELEMENT("TotalNetAssetValue", + XMLELEMENT("Amount", + XMLATTRIBUTES(f.currency AS "ccy"), + TO_CHAR(f.total_nav,'FM999999990.00'))))), + XMLELEMENT("Portfolios", + (SELECT XMLAGG( + XMLELEMENT("Portfolio", + XMLELEMENT("NavDate", + TO_CHAR(pf.nav_date,'YYYY-MM-DD')), + XMLELEMENT("Positions", + (SELECT XMLAGG( + XMLELEMENT("Position", + XMLFOREST(p.unique_id AS "UniqueID", + p.currency AS "Currency"), + XMLELEMENT("TotalValue", + XMLELEMENT("Amount", + XMLATTRIBUTES(f.currency AS "ccy"), + TO_CHAR(p.value_fund_ccy,'FM999999990.00'))), + XMLELEMENT("TotalPercentage", + TO_CHAR(p.percentage,'FM990.00')), + XMLELEMENT(EVALNAME p.kind)) + ORDER BY p.position_seq) + FROM position p + WHERE p.document_id=f.document_id + AND p.fund_seq=f.fund_seq + AND p.portfolio_seq=pf.portfolio_seq))) + ORDER BY pf.portfolio_seq) + FROM portfolio pf + WHERE pf.document_id=f.document_id + AND pf.fund_seq=f.fund_seq)))) + ORDER BY f.fund_seq) + FROM fund f WHERE f.document_id = :doc)) + ).getclobval() +FROM dual; --- As with the Postgres example, add the quantity child per kind --- (position.kind_qty) to keep Equity/Bond/ShareClass/... schema-valid. +-- Position class element emitted empty for brevity; add the quantity child +-- (position.kind_qty) and the per-fund SingleFund/ShareClasses block as the +-- runnable examples do. diff --git a/Database_Integration/generate_fundsxml/postgres.sql b/Database_Integration/generate_fundsxml/postgres.sql index b7d2833..88b7754 100644 --- a/Database_Integration/generate_fundsxml/postgres.sql +++ b/Database_Integration/generate_fundsxml/postgres.sql @@ -1,64 +1,82 @@ --- PostgreSQL -> FundsXML (code reference; no DB is provisioned in this repo). +-- PostgreSQL -> FundsXML, MULTI-FUND (code reference; no DB is provisioned). -- --- SQL/XML publishing with xmlelement/xmlagg/xmlattributes. Produces the same --- FundsXML 4.2.9 positions shape the Python reference emits (XSD-valid). --- Parameter :doc = fund.document_id. +-- SQL/XML publishing. The key multi-node move: a correlated subquery with +-- xmlagg(... ORDER BY fund_seq) emits one per fund row, and inside it +-- another xmlagg over portfolio -> positions. Emits the same canonical shape +-- as the runnable examples (so xml_equiv.py treats it as equal). :doc binds +-- document.document_id. Constants the model does not store (TotalAssetNature= +-- OFFICIAL, etc.) are reproduced verbatim. See ../ddl/schema.sql. SELECT xmlelement(name "FundsXML4", - xmlattributes('http://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd' - AS "xsi:noNamespaceSchemaLocation"), - xmlelement(name "ControlData", - xmlelement(name "UniqueDocumentID", f.document_id), - xmlelement(name "DocumentGenerated", '2025-10-02T00:00:00'), - CASE WHEN f.fxml_version IS NOT NULL - THEN xmlelement(name "Version", f.fxml_version) END, - xmlelement(name "ContentDate", f.content_date), - xmlelement(name "DataSupplier", - xmlelement(name "SystemCountry",'AT'), - xmlelement(name "Short",'EURAM'), - xmlelement(name "Name",'Erste Asset Management GmbH'), - xmlelement(name "Type",'Asset Manager')), - xmlelement(name "DataOperation",'INITIAL')), + xmlattributes( + 'http://www.w3.org/2001/XMLSchema-instance' AS "xmlns:xsi", + 'https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd' + AS "xsi:noNamespaceSchemaLocation"), + -- ControlData + (SELECT xmlelement(name "ControlData", + xmlelement(name "UniqueDocumentID", doc.document_id), + xmlelement(name "DocumentGenerated", doc.generated), + CASE WHEN doc.version IS NOT NULL + THEN xmlelement(name "Version", doc.version) END, + xmlelement(name "ContentDate", doc.content_date), + xmlelement(name "DataSupplier", + xmlelement(name "SystemCountry", doc.supplier_country), + xmlelement(name "Short", doc.supplier_short), + xmlelement(name "Name", doc.supplier_name), + xmlelement(name "Type", doc.supplier_type)), + xmlelement(name "DataOperation", doc.data_operation)) + FROM document doc WHERE doc.document_id = :doc), + -- Funds: one per fund row, in fund_seq order xmlelement(name "Funds", - xmlelement(name "Fund", - xmlelement(name "Identifiers", - xmlelement(name "LEI", f.lei)), - xmlelement(name "Names", - xmlelement(name "OfficialName", f.official_name)), - xmlelement(name "Currency", f.currency), - xmlelement(name "SingleFundFlag",'true'), - xmlelement(name "FundDynamicData", - xmlelement(name "TotalAssetValues", - xmlelement(name "TotalAssetValue", - xmlelement(name "NavDate", f.nav_date), - xmlelement(name "TotalAssetNature",'OFFICIAL'), - xmlelement(name "TotalNetAssetValue", - xmlelement(name "Amount", - xmlattributes(f.currency AS "ccy"), - to_char(f.total_nav,'FM999999999990.00'))))), - xmlelement(name "Portfolios", - xmlelement(name "Portfolio", - xmlelement(name "NavDate", f.nav_date), - xmlelement(name "Positions", - (SELECT xmlagg( - xmlelement(name "Position", - xmlelement(name "UniqueID", p.unique_id), - CASE WHEN p.isin IS NOT NULL THEN - xmlelement(name "Identifiers", - xmlelement(name "ISIN", p.isin)) END, - xmlelement(name "Currency", p.currency), - xmlelement(name "TotalValue", - xmlelement(name "Amount", - xmlattributes(f.currency AS "ccy"), - to_char(p.value_fund_ccy,'FM999999999990.00'))), - xmlelement(name "TotalPercentage", - to_char(p.percentage,'FM990.00')), - xmlelement(name p.kind)) ORDER BY p.unique_id) - FROM position p WHERE p.document_id = f.document_id)))))) + (SELECT xmlagg( + xmlelement(name "Fund", + CASE WHEN f.lei IS NOT NULL THEN xmlelement(name "Identifiers", + xmlelement(name "LEI", f.lei)) END, + xmlelement(name "Names", + xmlelement(name "OfficialName", f.official_name)), + xmlelement(name "Currency", f.currency), + CASE WHEN f.single_fund_flag IS NOT NULL + THEN xmlelement(name "SingleFundFlag", + f.single_fund_flag) END, + xmlelement(name "FundDynamicData", + xmlelement(name "TotalAssetValues", + xmlelement(name "TotalAssetValue", + xmlelement(name "NavDate", f.nav_date), + xmlelement(name "TotalAssetNature", 'OFFICIAL'), + xmlelement(name "TotalNetAssetValue", + xmlelement(name "Amount", + xmlattributes(f.currency AS "ccy"), + to_char(f.total_nav,'FM999999990.00'))))), + xmlelement(name "Portfolios", + (SELECT xmlagg( + xmlelement(name "Portfolio", + xmlelement(name "NavDate", pf.nav_date), + xmlelement(name "Positions", + (SELECT xmlagg( + xmlelement(name "Position", + xmlelement(name "UniqueID", p.unique_id), + xmlelement(name "Currency", p.currency), + xmlelement(name "TotalValue", + xmlelement(name "Amount", + xmlattributes(f.currency AS "ccy"), + to_char(p.value_fund_ccy,'FM999999990.00'))), + xmlelement(name "TotalPercentage", + to_char(p.percentage,'FM990.00')), + xmlelement(name p.kind)) + ORDER BY p.position_seq) + FROM position p + WHERE p.document_id=f.document_id + AND p.fund_seq=f.fund_seq + AND p.portfolio_seq=pf.portfolio_seq))) + ORDER BY pf.portfolio_seq) + FROM portfolio pf + WHERE pf.document_id=f.document_id + AND pf.fund_seq=f.fund_seq)))) + ORDER BY f.fund_seq) + FROM fund f WHERE f.document_id = :doc)) ) -FROM fund f -WHERE f.document_id = :doc; +FROM (SELECT 1) _; --- For brevity the position class element is emitted empty; add the quantity --- child (Units/Nominal/Shares/Contracts from position.kind_qty) exactly as the --- Python reference does to keep Equity/Bond/ShareClass/... schema-valid. +-- The Position class element is emitted empty for brevity; add its quantity +-- child (position.kind_qty -> Units/Nominal/Shares/Contracts) and the +-- per-fund SingleFund/ShareClasses block exactly as the runnable examples do. diff --git a/Database_Integration/generate_fundsxml/sqlserver.sql b/Database_Integration/generate_fundsxml/sqlserver.sql index bb607c1..5aa7fb7 100644 --- a/Database_Integration/generate_fundsxml/sqlserver.sql +++ b/Database_Integration/generate_fundsxml/sqlserver.sql @@ -1,51 +1,61 @@ --- SQL Server -> FundsXML (code reference; no DB is provisioned in this repo). +-- SQL Server -> FundsXML, MULTI-FUND (code reference; no DB is provisioned). -- --- FOR XML PATH with nested subqueries. Same FundsXML 4.2.9 positions shape as --- the Python reference. @doc = fund.document_id. +-- FOR XML PATH with nested correlated subqueries. Multi-node essence: the +-- outer FOR XML PATH('Fund') over `fund` (ORDER BY fund_seq) yields many +-- ; a correlated FOR XML PATH('Portfolio') over `portfolio`, and inside +-- it FOR XML PATH('Position') over `position`, give the nested repetition. +-- Same canonical shape as the runnable examples (xml_equiv.py-equal). +-- @doc = document.document_id. Full mapping: ../ddl/schema.sql. -DECLARE @doc varchar(128) = 'FUNDSXML_FILE_1'; +DECLARE @doc varchar(128) = 'FUNDSXML_MULTI_1'; SELECT - 'http://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd' + 'http://www.w3.org/2001/XMLSchema-instance' AS [@xmlns:xsi], + 'https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd' AS [@xsi:noNamespaceSchemaLocation], - f.document_id AS [ControlData/UniqueDocumentID], - '2025-10-02T00:00:00' AS [ControlData/DocumentGenerated], - f.fxml_version AS [ControlData/Version], - CONVERT(varchar(10), f.content_date, 23) AS [ControlData/ContentDate], - 'AT' AS [ControlData/DataSupplier/SystemCountry], - 'EURAM' AS [ControlData/DataSupplier/Short], - 'Erste Asset Management GmbH' AS [ControlData/DataSupplier/Name], - 'Asset Manager' AS [ControlData/DataSupplier/Type], - 'INITIAL' AS [ControlData/DataOperation], - f.lei AS [Funds/Fund/Identifiers/LEI], - f.official_name AS [Funds/Fund/Names/OfficialName], - f.currency AS [Funds/Fund/Currency], - 'true' AS [Funds/Fund/SingleFundFlag], - CONVERT(varchar(10), f.nav_date, 23) - AS [Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/NavDate], - 'OFFICIAL' - AS [Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/TotalAssetNature], - f.currency - AS [Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount/@ccy], - CONVERT(varchar(32), f.total_nav) - AS [Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount], + (SELECT d.document_id AS [UniqueDocumentID], + d.generated AS [DocumentGenerated], + d.version AS [Version], + CONVERT(varchar(10), d.content_date, 23) AS [ContentDate], + d.supplier_country AS [DataSupplier/SystemCountry], + d.supplier_short AS [DataSupplier/Short], + d.supplier_name AS [DataSupplier/Name], + d.supplier_type AS [DataSupplier/Type], + d.data_operation AS [DataOperation] + FROM document d WHERE d.document_id=@doc + FOR XML PATH('ControlData'), TYPE), (SELECT - p.unique_id AS [UniqueID], - p.currency AS [Currency], - f.currency AS [TotalValue/Amount/@ccy], - CONVERT(varchar(32), p.value_fund_ccy) AS [TotalValue/Amount], - CONVERT(varchar(16), p.percentage) AS [TotalPercentage] - -- plus the kind element (+ quantity child) — see note below - FROM position p - WHERE p.document_id = f.document_id - ORDER BY p.unique_id - FOR XML PATH('Position'), TYPE) - AS [Funds/Fund/FundDynamicData/Portfolios/Portfolio/Positions] -FROM fund f -WHERE f.document_id = @doc + f.lei AS [Identifiers/LEI], + f.official_name AS [Names/OfficialName], + f.currency AS [Currency], + f.single_fund_flag AS [SingleFundFlag], + CONVERT(varchar(10), f.nav_date, 23) + AS [FundDynamicData/TotalAssetValues/TotalAssetValue/NavDate], + 'OFFICIAL' + AS [FundDynamicData/TotalAssetValues/TotalAssetValue/TotalAssetNature], + f.currency + AS [FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount/@ccy], + CONVERT(varchar(32), f.total_nav) + AS [FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount], + (SELECT CONVERT(varchar(10), pf.nav_date, 23) AS [NavDate], + (SELECT p.unique_id AS [UniqueID], p.currency AS [Currency], + f.currency AS [TotalValue/Amount/@ccy], + CONVERT(varchar(32), p.value_fund_ccy) AS [TotalValue/Amount], + CONVERT(varchar(16), p.percentage) AS [TotalPercentage] + FROM position p + WHERE p.document_id=f.document_id AND p.fund_seq=f.fund_seq + AND p.portfolio_seq=pf.portfolio_seq + ORDER BY p.position_seq + FOR XML PATH('Position'), TYPE) AS [Positions] + FROM portfolio pf + WHERE pf.document_id=f.document_id AND pf.fund_seq=f.fund_seq + ORDER BY pf.portfolio_seq + FOR XML PATH('Portfolio'), TYPE) AS [FundDynamicData/Portfolios] + FROM fund f WHERE f.document_id=@doc + ORDER BY f.fund_seq + FOR XML PATH('Fund'), ROOT('Funds'), TYPE) FOR XML PATH('FundsXML4'); --- The position class element (Equity/Bond/.../with its Units/Nominal/Shares/ --- Contracts child from position.kind_qty) is added with a correlated --- FOR XML PATH subquery per row — emitted dynamically because the element name --- itself varies; see the Python reference for the exact mapping. +-- The Position class element (+ its quantity child from position.kind_qty) +-- and the per-fund SingleFund/ShareClasses block are added with further +-- correlated FOR XML PATH subqueries, exactly as the runnable examples do. diff --git a/Database_Integration/java/FundsXmlDb.java b/Database_Integration/java/FundsXmlDb.java new file mode 100644 index 0000000..2d83b04 --- /dev/null +++ b/Database_Integration/java/FundsXmlDb.java @@ -0,0 +1,493 @@ +// ============================================================================= +// FundsXML <-> relational database — runnable reference (Java, SQLite via JDBC). +// +// PURPOSE +// Standalone, copy-me example showing BOTH directions (import a FundsXML file +// into the relational schema, and export it back out). Over-commented on +// purpose: read it once and reimplement the pattern in your own project. +// +// DB SCHEMA +// ../ddl/schema.sql (document -> fund -> portfolio -> position; share_class +// per fund; asset document-scoped). SQLite keeps the example zero-setup. +// +// RUN (the sqlite-jdbc jar is fetched by tools/fetch-tools.sh into .lib/) +// tools/fetch-tools.sh +// CP=.lib/sqlite-jdbc-3.46.1.3.jar +// javac -cp "$CP" -d /tmp/db Database_Integration/java/FundsXmlDb.java +// java -cp "$CP:/tmp/db" FundsXmlDb roundtrip \ +// FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml out.xml +// commands: init | import | export | +// roundtrip +// +// DEPENDENCIES +// org.xerial:sqlite-jdbc (self-contained, bundles native SQLite) + the JDK. +// XML uses native javax.xml DOM/XPath — NO JAXB (the FundsXML schema is huge; +// a thin DOM binding is simpler and version-tolerant). +// +// FUNDSXML ASSUMPTIONS (non-obvious) +// * FundsXML 4.x has NO XML namespace -> XPath uses bare element names. +// * Many , each many , each many : we iterate all +// and store 1-based *_seq ordinals so the export is order-faithful and the +// round-trip compares equal (see ../tools/xml_equiv.py). +// * Positions link to AssetMasterData by shared ; `asset` is +// therefore document-scoped, not per-fund. +// * Export is normalized to the 4.2.9 schema URL. Constants the model does +// not store (TotalAssetNature=OFFICIAL, Price ActionCode=C/PriceNature= +// OFFICIAL) are reproduced verbatim so the round-trip stays equal. +// +// SECURITY +// The DOM parser forbids DOCTYPE/external entities (XXE / billion laughs). +// ============================================================================= +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Locale; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +public class FundsXmlDb { + + static final String SCHEMA_URL = + "https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd"; + static final String XSI = "http://www.w3.org/2001/XMLSchema-instance"; + + // Position instrument-class element + its mandatory quantity child. + // Kinds absent from QTY_ELEM are schema-valid as an empty element. + static final java.util.Set POSITION_KINDS = java.util.Set.of( + "Equity", "Bond", "ShareClass", "Warrant", "Certificate", "Option", + "Future", "FXForward", "Swap", "Repo", "RealEstate", "CallMoney", + "Account", "Generic"); + static final java.util.Map QTY_ELEM = java.util.Map.of( + "Equity", "Units", "Warrant", "Units", "Certificate", "Units", + "Bond", "Nominal", "ShareClass", "Shares", + "Option", "Contracts", "Future", "Contracts"); + + // ---- helpers ----------------------------------------------------------- + static XPath xpath() { return XPathFactory.newInstance().newXPath(); } + + /** First matching descendant's text (bare element names; no namespace). */ + static String t(Node ctx, String expr) throws Exception { + Node n = (Node) xpath().evaluate(expr, ctx, XPathConstants.NODE); + return n == null ? null : n.getTextContent(); + } + + static Double num(String s) { + return (s == null || s.isBlank()) ? null : Double.valueOf(s.trim()); + } + + static String f2(double v) { return String.format(Locale.ROOT, "%.2f", v); } + + /** Append text to parent (the single XML-build primitive). */ + static Element el(Document doc, Node parent, String tag, String text) { + Element e = doc.createElement(tag); + if (text != null) e.setTextContent(text); + parent.appendChild(e); + return e; + } + + static Connection open(String db) throws Exception { + Connection c = DriverManager.getConnection("jdbc:sqlite:" + db); + c.createStatement().execute("PRAGMA foreign_keys = ON"); + return c; + } + + static Document parseSecure(String xml) throws Exception { + DocumentBuilderFactory f = DocumentBuilderFactory.newInstance(); + f.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + f.setFeature("http://xml.org/sax/features/external-general-entities", false); + f.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + f.setExpandEntityReferences(false); + try (FileInputStream in = new FileInputStream(xml)) { + return f.newDocumentBuilder().parse(in); + } + } + + // ---- init -------------------------------------------------------------- + static void init(String db) throws Exception { + Path ddl = Path.of(System.getProperty("user.dir"), + "Database_Integration", "ddl", "schema.sql"); + // Fallback when run from inside Database_Integration/java. + if (!Files.exists(ddl)) + ddl = Path.of("..", "ddl", "schema.sql"); + // Strip "--" line comments first (the DDL has no string literals, so a + // plain cut at "--" is safe) so only real SQL statements remain. + StringBuilder clean = new StringBuilder(); + for (String line : Files.readString(ddl).split("\n")) { + int cut = line.indexOf("--"); + clean.append(cut >= 0 ? line.substring(0, cut) : line).append('\n'); + } + try (Connection c = open(db); Statement st = c.createStatement()) { + for (String stmt : clean.toString().split(";")) { + if (!stmt.strip().isEmpty()) st.execute(stmt); + } + } + } + + // ---- import : FundsXML -> rows ----------------------------------------- + static String doImport(String db, String xml) throws Exception { + Document doc = parseSecure(xml); + XPath xp = xpath(); + Node cd = (Node) xp.evaluate("/FundsXML4/ControlData", doc, + XPathConstants.NODE); + String docId = t(cd, "UniqueDocumentID"); + + try (Connection c = open(db)) { + c.setAutoCommit(false); + + try (PreparedStatement ps = c.prepareStatement( + "INSERT INTO document VALUES (?,?,?,?,?,?,?,?,?)")) { + ps.setString(1, docId); + ps.setString(2, t(cd, "DocumentGenerated")); + ps.setString(3, t(cd, "Version")); + ps.setString(4, t(cd, "ContentDate")); + ps.setString(5, t(cd, "DataOperation")); + ps.setString(6, t(cd, "DataSupplier/SystemCountry")); + ps.setString(7, t(cd, "DataSupplier/Short")); + ps.setString(8, t(cd, "DataSupplier/Name")); + ps.setString(9, t(cd, "DataSupplier/Type")); + ps.executeUpdate(); + } + + NodeList funds = (NodeList) xp.evaluate("/FundsXML4/Funds/Fund", + doc, XPathConstants.NODESET); + for (int fi = 0; fi < funds.getLength(); fi++) { + Node fund = funds.item(fi); + int fundSeq = fi + 1; // 1-based document order + String ccy = t(fund, "Currency"); + Node tav = (Node) xp.evaluate( + "FundDynamicData/TotalAssetValues/TotalAssetValue", + fund, XPathConstants.NODE); + + try (PreparedStatement ps = c.prepareStatement( + "INSERT INTO fund VALUES (?,?,?,?,?,?,?,?)")) { + ps.setString(1, docId); + ps.setInt(2, fundSeq); + ps.setString(3, t(fund, "Identifiers/LEI")); + ps.setString(4, t(fund, "Names/OfficialName")); + ps.setString(5, ccy); + ps.setString(6, t(fund, "SingleFundFlag")); + ps.setString(7, t(tav, "NavDate")); + ps.setDouble(8, Double.parseDouble(t(tav, + "TotalNetAssetValue/Amount[@ccy='" + ccy + "']"))); + ps.executeUpdate(); + } + + NodeList scs = (NodeList) xp.evaluate( + "SingleFund/ShareClasses/ShareClass", fund, + XPathConstants.NODESET); + for (int si = 0; si < scs.getLength(); si++) { + Node sc = scs.item(si); + try (PreparedStatement ps = c.prepareStatement( + "INSERT INTO share_class VALUES (?,?,?,?,?,?,?,?)")) { + ps.setString(1, docId); + ps.setInt(2, fundSeq); + ps.setString(3, t(sc, "Identifiers/ISIN")); + ps.setString(4, t(sc, "Names/OfficialName")); + ps.setString(5, t(sc, "Currency")); + setNum(ps, 6, num(t(sc, "Prices/Price/NavPrice"))); + setNum(ps, 7, num(t(sc, "TotalAssetValues/" + + "TotalAssetValue/TotalNetAssetValue/Amount[@ccy='" + + ccy + "']"))); + setNum(ps, 8, num(t(sc, "TotalAssetValues/" + + "TotalAssetValue/SharesOutstanding"))); + ps.executeUpdate(); + } + } + + NodeList ports = (NodeList) xp.evaluate( + "FundDynamicData/Portfolios/Portfolio", fund, + XPathConstants.NODESET); + for (int pi = 0; pi < ports.getLength(); pi++) { + Node port = ports.item(pi); + int portSeq = pi + 1; + try (PreparedStatement ps = c.prepareStatement( + "INSERT INTO portfolio VALUES (?,?,?,?)")) { + ps.setString(1, docId); + ps.setInt(2, fundSeq); + ps.setInt(3, portSeq); + ps.setString(4, t(port, "NavDate")); + ps.executeUpdate(); + } + NodeList poss = (NodeList) xp.evaluate( + "Positions/Position", port, XPathConstants.NODESET); + for (int qi = 0; qi < poss.getLength(); qi++) { + Node pos = poss.item(qi); + String kind = null; + for (Node ch = pos.getFirstChild(); ch != null; + ch = ch.getNextSibling()) + if (ch.getNodeType() == Node.ELEMENT_NODE + && POSITION_KINDS.contains(ch.getNodeName())) { + kind = ch.getNodeName(); + break; + } + Double qty = (kind != null && QTY_ELEM.containsKey(kind)) + ? num(t(pos, kind + "/" + QTY_ELEM.get(kind))) + : null; + try (PreparedStatement ps = c.prepareStatement( + "INSERT INTO position VALUES (?,?,?,?,?,?,?,?,?,?,?)")) { + ps.setString(1, docId); + ps.setInt(2, fundSeq); + ps.setInt(3, portSeq); + ps.setInt(4, qi + 1); + ps.setString(5, t(pos, "UniqueID")); + ps.setString(6, t(pos, "Identifiers/ISIN")); + ps.setString(7, t(pos, "Currency")); + ps.setDouble(8, Double.parseDouble(t(pos, + "TotalValue/Amount[@ccy='" + ccy + "']"))); + ps.setDouble(9, Double.parseDouble( + t(pos, "TotalPercentage"))); + ps.setString(10, kind); + setNum(ps, 11, qty); + ps.executeUpdate(); + } + } + } + } + + NodeList assets = (NodeList) xp.evaluate( + "/FundsXML4/AssetMasterData/Asset", doc, XPathConstants.NODESET); + for (int ai = 0; ai < assets.getLength(); ai++) { + Node a = assets.item(ai); + try (PreparedStatement ps = c.prepareStatement( + "INSERT INTO asset VALUES (?,?,?,?,?,?,?)")) { + ps.setString(1, docId); + ps.setString(2, t(a, "UniqueID")); + ps.setString(3, t(a, "Identifiers/ISIN")); + ps.setString(4, t(a, "Name")); + ps.setString(5, t(a, "AssetType")); + ps.setString(6, t(a, "Currency")); + ps.setString(7, t(a, "Country")); + ps.executeUpdate(); + } + } + c.commit(); + } + return docId; + } + + static void setNum(PreparedStatement ps, int i, Double v) throws Exception { + if (v == null) ps.setNull(i, java.sql.Types.DECIMAL); + else ps.setDouble(i, v); + } + + // ---- export : rows -> FundsXML ---------------------------------------- + static void export(String db, String docId, String out) throws Exception { + Document doc = DocumentBuilderFactory.newInstance() + .newDocumentBuilder().newDocument(); + Element root = doc.createElement("FundsXML4"); + root.setAttribute("xmlns:xsi", XSI); + root.setAttribute("xsi:noNamespaceSchemaLocation", SCHEMA_URL); + doc.appendChild(root); + + try (Connection c = open(db)) { + ResultSet d = c.createStatement().executeQuery( + "SELECT * FROM document WHERE document_id='" + docId + "'"); + if (!d.next()) throw new RuntimeException("no document " + docId); + + Element cd = el(doc, root, "ControlData", null); + el(doc, cd, "UniqueDocumentID", d.getString("document_id")); + // Regenerate timestamp; xml_equiv.py ignores its value. + String gen = d.getString("generated"); + el(doc, cd, "DocumentGenerated", + gen != null ? gen : "2025-10-02T00:00:00"); + String ver = d.getString("version"); + if (ver != null) el(doc, cd, "Version", ver); // none for 4.0.0 + el(doc, cd, "ContentDate", d.getString("content_date")); + Element ds = el(doc, cd, "DataSupplier", null); + el(doc, ds, "SystemCountry", d.getString("supplier_country")); + el(doc, ds, "Short", d.getString("supplier_short")); + el(doc, ds, "Name", d.getString("supplier_name")); + el(doc, ds, "Type", d.getString("supplier_type")); + el(doc, cd, "DataOperation", d.getString("data_operation")); + + Element fundsEl = el(doc, root, "Funds", null); + PreparedStatement fps = c.prepareStatement( + "SELECT * FROM fund WHERE document_id=? ORDER BY fund_seq"); + fps.setString(1, docId); + ResultSet fr = fps.executeQuery(); + while (fr.next()) { + int fundSeq = fr.getInt("fund_seq"); + String ccy = fr.getString("currency"); + String navDate = fr.getString("nav_date"); + Element fund = el(doc, fundsEl, "Fund", null); + if (fr.getString("lei") != null) + el(doc, el(doc, fund, "Identifiers", null), "LEI", + fr.getString("lei")); + el(doc, el(doc, fund, "Names", null), "OfficialName", + fr.getString("official_name")); + el(doc, fund, "Currency", ccy); + if (fr.getString("single_fund_flag") != null) + el(doc, fund, "SingleFundFlag", + fr.getString("single_fund_flag")); + + Element fdd = el(doc, fund, "FundDynamicData", null); + Element tav = el(doc, el(doc, el(doc, fdd, + "TotalAssetValues", null), "TotalAssetValue", null), + "NavDate", navDate); + Element tavp = (Element) tav.getParentNode(); + el(doc, tavp, "TotalAssetNature", "OFFICIAL"); + Element tnav = el(doc, tavp, "TotalNetAssetValue", null); + Element amt = el(doc, tnav, "Amount", + f2(fr.getDouble("total_nav"))); + amt.setAttribute("ccy", ccy); + + Element ports = el(doc, fdd, "Portfolios", null); + PreparedStatement pps = c.prepareStatement( + "SELECT * FROM portfolio WHERE document_id=? AND fund_seq=?" + + " ORDER BY portfolio_seq"); + pps.setString(1, docId); pps.setInt(2, fundSeq); + ResultSet pr = pps.executeQuery(); + while (pr.next()) { + int portSeq = pr.getInt("portfolio_seq"); + Element pe = el(doc, ports, "Portfolio", null); + el(doc, pe, "NavDate", pr.getString("nav_date")); + Element poss = el(doc, pe, "Positions", null); + PreparedStatement qps = c.prepareStatement( + "SELECT * FROM position WHERE document_id=? AND " + + "fund_seq=? AND portfolio_seq=? ORDER BY position_seq"); + qps.setString(1, docId); qps.setInt(2, fundSeq); + qps.setInt(3, portSeq); + ResultSet qr = qps.executeQuery(); + while (qr.next()) { + Element pos = el(doc, poss, "Position", null); + el(doc, pos, "UniqueID", qr.getString("unique_id")); + if (qr.getString("isin") != null) + el(doc, el(doc, pos, "Identifiers", null), "ISIN", + qr.getString("isin")); + if (qr.getString("currency") != null) + el(doc, pos, "Currency", qr.getString("currency")); + Element tv = el(doc, el(doc, pos, "TotalValue", null), + "Amount", f2(qr.getDouble("value_fund_ccy"))); + tv.setAttribute("ccy", ccy); + el(doc, pos, "TotalPercentage", + f2(qr.getDouble("percentage"))); + String kind = qr.getString("kind"); + if (kind == null || !POSITION_KINDS.contains(kind)) + kind = "Generic"; + Element ke = el(doc, pos, kind, null); + Object q = qr.getObject("kind_qty"); + if (QTY_ELEM.containsKey(kind) && q != null) + el(doc, ke, QTY_ELEM.get(kind), + f2(((Number) q).doubleValue())); + } + } + + PreparedStatement sps = c.prepareStatement( + "SELECT * FROM share_class WHERE document_id=? AND " + + "fund_seq=? ORDER BY isin"); + sps.setString(1, docId); sps.setInt(2, fundSeq); + ResultSet sr = sps.executeQuery(); + Element sce = null; + while (sr.next()) { + if (sce == null) + sce = el(doc, el(doc, fund, "SingleFund", null), + "ShareClasses", null); + Element x = el(doc, sce, "ShareClass", null); + el(doc, el(doc, x, "Identifiers", null), "ISIN", + sr.getString("isin")); + if (sr.getString("official_name") != null) + el(doc, el(doc, x, "Names", null), "OfficialName", + sr.getString("official_name")); + el(doc, x, "Currency", sr.getString("currency")); + Object navp = sr.getObject("nav_price"); + if (navp != null) { + Element pe2 = el(doc, el(doc, x, "Prices", null), + "Price", null); + el(doc, pe2, "ActionCode", "C"); + el(doc, pe2, "NavDate", navDate); + el(doc, pe2, "PriceCurrency", sr.getString("currency")); + el(doc, pe2, "PriceNature", "OFFICIAL"); + el(doc, pe2, "NavPrice", + f2(((Number) navp).doubleValue())); + } + Object navf = sr.getObject("nav_fund_ccy"); + if (navf != null) { + Element t2 = el(doc, el(doc, x, "TotalAssetValues", + null), "TotalAssetValue", null); + el(doc, t2, "NavDate", navDate); + el(doc, t2, "TotalAssetNature", "OFFICIAL"); + Element a2 = el(doc, el(doc, t2, + "TotalNetAssetValue", null), "Amount", + f2(((Number) navf).doubleValue())); + a2.setAttribute("ccy", ccy); + Object so = sr.getObject("shares_outstanding"); + if (so != null) + el(doc, t2, "SharesOutstanding", String.format( + Locale.ROOT, "%.0f", + ((Number) so).doubleValue())); + } + } + } + + PreparedStatement aps = c.prepareStatement( + "SELECT * FROM asset WHERE document_id=? ORDER BY unique_id"); + aps.setString(1, docId); + ResultSet ar = aps.executeQuery(); + Element amd = null; + while (ar.next()) { + if (amd == null) amd = el(doc, root, "AssetMasterData", null); + Element ae = el(doc, amd, "Asset", null); + el(doc, ae, "UniqueID", ar.getString("unique_id")); + if (ar.getString("isin") != null) + el(doc, el(doc, ae, "Identifiers", null), "ISIN", + ar.getString("isin")); + el(doc, ae, "Currency", ar.getString("currency")); + if (ar.getString("country") != null) + el(doc, ae, "Country", ar.getString("country")); + el(doc, ae, "Name", ar.getString("name")); + el(doc, ae, "AssetType", ar.getString("asset_type")); + } + } + + Transformer tr = TransformerFactory.newInstance().newTransformer(); + tr.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); + tr.setOutputProperty(OutputKeys.INDENT, "yes"); + try (FileOutputStream os = new FileOutputStream(out)) { + tr.transform(new DOMSource(doc), new StreamResult(os)); + } + } + + // ---- cli --------------------------------------------------------------- + public static void main(String[] args) throws Exception { + if (args.length == 0) { System.err.println( + "usage: init|import|export|roundtrip ..."); System.exit(2); } + switch (args[0]) { + case "init" -> init(args[1]); + case "import" -> + System.out.println("imported " + doImport(args[1], args[2])); + case "export" -> { export(args[1], args[2], args[3]); + System.out.println("wrote " + args[3]); } + case "roundtrip" -> { + // import then export THROUGH the DB (the required test). + File tmp = File.createTempFile("fxdb", ".db"); + tmp.delete(); + init(tmp.getPath()); + String id = doImport(tmp.getPath(), args[1]); + export(tmp.getPath(), id, args[2]); + tmp.delete(); + System.out.println("round-trip ok: " + args[1] + + " -> DB -> " + args[2] + " (doc " + id + ")"); + } + default -> { System.err.println("unknown: " + args[0]); + System.exit(2); } + } + } +} diff --git a/Database_Integration/javascript/fundsxml_db.mjs b/Database_Integration/javascript/fundsxml_db.mjs new file mode 100644 index 0000000..dbd81bb --- /dev/null +++ b/Database_Integration/javascript/fundsxml_db.mjs @@ -0,0 +1,317 @@ +// ============================================================================= +// FundsXML <-> relational database — runnable reference (Node.js, SQLite). +// +// PURPOSE +// Standalone, copy-me example of BOTH directions (import a FundsXML file into +// the relational schema, export it back). Over-commented on purpose. +// +// DB SCHEMA ../ddl/schema.sql (document -> fund -> portfolio -> position; +// share_class per fund; asset document-scoped). +// +// RUN +// cd Database_Integration/javascript && npm install +// node fundsxml_db.mjs roundtrip \ +// ../../FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml out.xml +// commands: init | import | export | +// roundtrip +// +// DEPENDENCIES (all pure-JS, no native build — runs on any Node >= 18): +// sql.js SQLite compiled to WebAssembly (the DB engine) +// @xmldom/xmldom a namespace-free DOM (FundsXML 4.x has no namespace) +// xpath XPath 1.0 over that DOM +// Node 20 has no built-in node:sqlite (that arrived in 22.5), hence sql.js. +// +// FUNDSXML ASSUMPTIONS +// * No XML namespace -> bare element names in XPath. +// * Many //: all iterated; 1-based *_seq columns +// preserve order so the round-trip compares equal (../tools/xml_equiv.py). +// * Positions link to AssetMasterData by shared -> `asset` is +// document-scoped. +// * Export normalized to the 4.2.9 schema URL; constants the model does not +// store (TotalAssetNature=OFFICIAL, Price ActionCode=C / PriceNature= +// OFFICIAL) are reproduced verbatim. +// +// SECURITY +// @xmldom/xmldom does not resolve external entities/DTDs (no XXE surface). +// ============================================================================= +import { readFileSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import initSqlJs from "sql.js"; +import { DOMParser, XMLSerializer } from "@xmldom/xmldom"; +import xpath from "xpath"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const DDL = resolve(HERE, "..", "ddl", "schema.sql"); +const SCHEMA_URL = + "https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd"; + +// Position instrument-class element + its mandatory quantity child. Kinds not +// in QTY_ELEM are schema-valid as an empty element. +const POSITION_KINDS = new Set(["Equity", "Bond", "ShareClass", "Warrant", + "Certificate", "Option", "Future", "FXForward", "Swap", "Repo", + "RealEstate", "CallMoney", "Account", "Generic"]); +const QTY_ELEM = { Equity: "Units", Warrant: "Units", Certificate: "Units", + Bond: "Nominal", ShareClass: "Shares", Option: "Contracts", + Future: "Contracts" }; + +// ---- tiny helpers ---------------------------------------------------------- +const txt = (ctx, expr) => { // first match's text or null + const n = xpath.select1(expr, ctx); + return n && n.textContent != null && n.textContent !== "" ? n.textContent : null; +}; +const numOrNull = (s) => (s == null || s === "" ? null : Number(s)); +const f2 = (v) => Number(v).toFixed(2); + +function el(doc, parent, tag, text) { // the one XML-build primitive + const e = doc.createElement(tag); + if (text != null) e.appendChild(doc.createTextNode(String(text))); + parent.appendChild(e); + return e; +} + +function execScript(db, sql) { + // Strip "--" line comments (no string literals in the DDL) then run. + const clean = sql.split("\n").map((l) => { + const i = l.indexOf("--"); + return i >= 0 ? l.slice(0, i) : l; + }).join("\n"); + for (const stmt of clean.split(";")) if (stmt.trim()) db.run(stmt); +} + +function parseSecure(xmlPath) { + // @xmldom/xmldom never expands external entities; we additionally reject a + // DOCTYPE so a malicious feed cannot even declare entities. + const src = readFileSync(xmlPath, "utf8"); + if (/ rows -------------------------------------------- +function doImport(db, xmlPath) { + const doc = parseSecure(xmlPath); + const cd = xpath.select1("/FundsXML4/ControlData", doc); + const docId = txt(cd, "UniqueDocumentID"); + + db.run("INSERT INTO document VALUES (?,?,?,?,?,?,?,?,?)", [ + docId, txt(cd, "DocumentGenerated"), txt(cd, "Version"), + txt(cd, "ContentDate"), txt(cd, "DataOperation"), + txt(cd, "DataSupplier/SystemCountry"), txt(cd, "DataSupplier/Short"), + txt(cd, "DataSupplier/Name"), txt(cd, "DataSupplier/Type")]); + + const funds = xpath.select("/FundsXML4/Funds/Fund", doc); + funds.forEach((fund, fi) => { + const fundSeq = fi + 1; // 1-based document order + const ccy = txt(fund, "Currency"); + const tav = xpath.select1( + "FundDynamicData/TotalAssetValues/TotalAssetValue", fund); + db.run("INSERT INTO fund VALUES (?,?,?,?,?,?,?,?)", [ + docId, fundSeq, txt(fund, "Identifiers/LEI"), + txt(fund, "Names/OfficialName"), ccy, txt(fund, "SingleFundFlag"), + txt(tav, "NavDate"), + Number(txt(tav, `TotalNetAssetValue/Amount[@ccy='${ccy}']`))]); + + for (const sc of xpath.select("SingleFund/ShareClasses/ShareClass", fund)) { + db.run("INSERT INTO share_class VALUES (?,?,?,?,?,?,?,?)", [ + docId, fundSeq, txt(sc, "Identifiers/ISIN"), + txt(sc, "Names/OfficialName"), txt(sc, "Currency"), + numOrNull(txt(sc, "Prices/Price/NavPrice")), + numOrNull(txt(sc, + `TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount[@ccy='${ccy}']`)), + numOrNull(txt(sc, + "TotalAssetValues/TotalAssetValue/SharesOutstanding"))]); + } + + xpath.select("FundDynamicData/Portfolios/Portfolio", fund) + .forEach((port, pi) => { + const portSeq = pi + 1; + db.run("INSERT INTO portfolio VALUES (?,?,?,?)", + [docId, fundSeq, portSeq, txt(port, "NavDate")]); + xpath.select("Positions/Position", port).forEach((pos, qi) => { + let kind = null; + for (let ch = pos.firstChild; ch; ch = ch.nextSibling) + if (ch.nodeType === 1 && POSITION_KINDS.has(ch.nodeName)) { + kind = ch.nodeName; break; + } + const qty = kind && QTY_ELEM[kind] + ? numOrNull(txt(pos, `${kind}/${QTY_ELEM[kind]}`)) : null; + db.run("INSERT INTO position VALUES (?,?,?,?,?,?,?,?,?,?,?)", [ + docId, fundSeq, portSeq, qi + 1, txt(pos, "UniqueID"), + txt(pos, "Identifiers/ISIN"), txt(pos, "Currency"), + Number(txt(pos, `TotalValue/Amount[@ccy='${ccy}']`)), + Number(txt(pos, "TotalPercentage")), kind, qty]); + }); + }); + }); + + for (const a of xpath.select("/FundsXML4/AssetMasterData/Asset", doc)) { + db.run("INSERT INTO asset VALUES (?,?,?,?,?,?,?)", [ + docId, txt(a, "UniqueID"), txt(a, "Identifiers/ISIN"), + txt(a, "Name"), txt(a, "AssetType"), txt(a, "Currency"), + txt(a, "Country")]); + } + return docId; +} + +// ---- export : rows -> FundsXML -------------------------------------------- +function rows(db, sql, params) { + const st = db.prepare(sql); + st.bind(params); + const out = []; + while (st.step()) out.push(st.getAsObject()); + st.free(); + return out; +} + +function exportXml(db, docId, outPath) { + const d = rows(db, "SELECT * FROM document WHERE document_id=?", [docId])[0]; + if (!d) throw new Error(`no document ${docId}`); + + const doc = new DOMParser().parseFromString("", "text/xml"); + const root = doc.documentElement; + root.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); + root.setAttribute("xsi:noNamespaceSchemaLocation", SCHEMA_URL); + + const cd = el(doc, root, "ControlData"); + el(doc, cd, "UniqueDocumentID", d.document_id); + // Regenerate the timestamp; xml_equiv.py ignores its value. + el(doc, cd, "DocumentGenerated", d.generated || "2025-10-02T00:00:00"); + if (d.version != null) el(doc, cd, "Version", d.version); // none for 4.0.0 + el(doc, cd, "ContentDate", d.content_date); + const ds = el(doc, cd, "DataSupplier"); + el(doc, ds, "SystemCountry", d.supplier_country); + el(doc, ds, "Short", d.supplier_short); + el(doc, ds, "Name", d.supplier_name); + el(doc, ds, "Type", d.supplier_type); + el(doc, cd, "DataOperation", d.data_operation); + + const fundsEl = el(doc, root, "Funds"); + for (const f of rows(db, + "SELECT * FROM fund WHERE document_id=? ORDER BY fund_seq", [docId])) { + const ccy = f.currency; + const fund = el(doc, fundsEl, "Fund"); + if (f.lei != null) el(doc, el(doc, fund, "Identifiers"), "LEI", f.lei); + el(doc, el(doc, fund, "Names"), "OfficialName", f.official_name); + el(doc, fund, "Currency", ccy); + if (f.single_fund_flag != null) + el(doc, fund, "SingleFundFlag", f.single_fund_flag); + + const fdd = el(doc, fund, "FundDynamicData"); + const tavp = el(doc, el(doc, fdd, "TotalAssetValues"), "TotalAssetValue"); + el(doc, tavp, "NavDate", f.nav_date); + el(doc, tavp, "TotalAssetNature", "OFFICIAL"); + const amt = el(doc, el(doc, tavp, "TotalNetAssetValue"), "Amount", + f2(f.total_nav)); + amt.setAttribute("ccy", ccy); + + const ports = el(doc, fdd, "Portfolios"); + for (const pf of rows(db, + "SELECT * FROM portfolio WHERE document_id=? AND fund_seq=? " + + "ORDER BY portfolio_seq", [docId, f.fund_seq])) { + const pe = el(doc, ports, "Portfolio"); + el(doc, pe, "NavDate", pf.nav_date); + const poss = el(doc, pe, "Positions"); + for (const p of rows(db, + "SELECT * FROM position WHERE document_id=? AND fund_seq=? AND " + + "portfolio_seq=? ORDER BY position_seq", + [docId, f.fund_seq, pf.portfolio_seq])) { + const pos = el(doc, poss, "Position"); + el(doc, pos, "UniqueID", p.unique_id); + if (p.isin != null) + el(doc, el(doc, pos, "Identifiers"), "ISIN", p.isin); + if (p.currency != null) el(doc, pos, "Currency", p.currency); + const tv = el(doc, el(doc, pos, "TotalValue"), "Amount", + f2(p.value_fund_ccy)); + tv.setAttribute("ccy", ccy); + el(doc, pos, "TotalPercentage", f2(p.percentage)); + const kind = POSITION_KINDS.has(p.kind) ? p.kind : "Generic"; + const ke = el(doc, pos, kind); + if (QTY_ELEM[kind] && p.kind_qty != null) + el(doc, ke, QTY_ELEM[kind], f2(p.kind_qty)); + } + } + + const scs = rows(db, "SELECT * FROM share_class WHERE document_id=? AND " + + "fund_seq=? ORDER BY isin", [docId, f.fund_seq]); + if (scs.length) { + const sce = el(doc, el(doc, fund, "SingleFund"), "ShareClasses"); + for (const sc of scs) { + const x = el(doc, sce, "ShareClass"); + el(doc, el(doc, x, "Identifiers"), "ISIN", sc.isin); + if (sc.official_name != null) + el(doc, el(doc, x, "Names"), "OfficialName", sc.official_name); + el(doc, x, "Currency", sc.currency); + if (sc.nav_price != null) { + const pr = el(doc, el(doc, x, "Prices"), "Price"); + el(doc, pr, "ActionCode", "C"); + el(doc, pr, "NavDate", f.nav_date); + el(doc, pr, "PriceCurrency", sc.currency); + el(doc, pr, "PriceNature", "OFFICIAL"); + el(doc, pr, "NavPrice", f2(sc.nav_price)); + } + if (sc.nav_fund_ccy != null) { + const t2 = el(doc, el(doc, x, "TotalAssetValues"), + "TotalAssetValue"); + el(doc, t2, "NavDate", f.nav_date); + el(doc, t2, "TotalAssetNature", "OFFICIAL"); + const a2 = el(doc, el(doc, t2, "TotalNetAssetValue"), "Amount", + f2(sc.nav_fund_ccy)); + a2.setAttribute("ccy", ccy); + if (sc.shares_outstanding != null) + el(doc, t2, "SharesOutstanding", + Number(sc.shares_outstanding).toFixed(0)); + } + } + } + } + + const assets = rows(db, + "SELECT * FROM asset WHERE document_id=? ORDER BY unique_id", [docId]); + if (assets.length) { + const amd = el(doc, root, "AssetMasterData"); + for (const a of assets) { + const ae = el(doc, amd, "Asset"); + el(doc, ae, "UniqueID", a.unique_id); + if (a.isin != null) + el(doc, el(doc, ae, "Identifiers"), "ISIN", a.isin); + el(doc, ae, "Currency", a.currency); + if (a.country != null) el(doc, ae, "Country", a.country); + el(doc, ae, "Name", a.name); + el(doc, ae, "AssetType", a.asset_type); + } + } + + const xml = '\n' + + new XMLSerializer().serializeToString(doc); + writeFileSync(outPath, xml); +} + +// ---- cli ------------------------------------------------------------------- +const [cmd, a1, a2, a3] = process.argv.slice(2); +const SQL = await initSqlJs(); +if (cmd === "init") { + const db = new SQL.Database(); + execScript(db, readFileSync(DDL, "utf8")); + persist(db, a1); +} else if (cmd === "import") { + const db = load(SQL, a1); + console.log("imported", doImport(db, a2)); + persist(db, a1); +} else if (cmd === "export") { + exportXml(load(SQL, a1), a2, a3); + console.log("wrote", a3); +} else if (cmd === "roundtrip") { + // import then export THROUGH an (in-memory) DB — the required test. + const db = new SQL.Database(); + execScript(db, readFileSync(DDL, "utf8")); + const id = doImport(db, a1); + exportXml(db, id, a2); + console.log(`round-trip ok: ${a1} -> DB -> ${a2} (doc ${id})`); +} else { + console.error("usage: init|import|export|roundtrip ..."); + process.exit(2); +} diff --git a/Database_Integration/javascript/package.json b/Database_Integration/javascript/package.json new file mode 100644 index 0000000..4a2037f --- /dev/null +++ b/Database_Integration/javascript/package.json @@ -0,0 +1,16 @@ +{ + "name": "fundsxml-db-example", + "version": "1.0.0", + "private": true, + "type": "module", + "description": "Standalone FundsXML <-> SQLite round-trip example (Node.js).", + "//": "Pure-JS deps so it runs anywhere with no native build: sql.js is SQLite compiled to WebAssembly; @xmldom/xmldom + xpath give a namespace-free DOM. Run: npm install && node fundsxml_db.mjs roundtrip ", + "scripts": { + "roundtrip": "node fundsxml_db.mjs roundtrip" + }, + "dependencies": { + "@xmldom/xmldom": "^0.9.8", + "sql.js": "^1.13.0", + "xpath": "^0.0.34" + } +} diff --git a/Database_Integration/load_from_fundsxml/oracle.sql b/Database_Integration/load_from_fundsxml/oracle.sql index 6b7e8a8..3c9461f 100644 --- a/Database_Integration/load_from_fundsxml/oracle.sql +++ b/Database_Integration/load_from_fundsxml/oracle.sql @@ -1,48 +1,66 @@ --- FundsXML -> Oracle (code reference; no DB is provisioned in this repo). +-- FundsXML -> Oracle, MULTI-FUND (code reference; no DB is provisioned). -- --- Strategy: stage in an XMLType column, shred with XMLTABLE. FundsXML 4.x has --- no namespace. Schema: ../ddl/schema.sql with VARCHAR2/NUMBER types. +-- Pattern: stage in an XMLType column; nested XMLTABLE with FOR ORDINALITY +-- gives fund_seq / portfolio_seq / position_seq (PASSING the parent node into +-- the inner XMLTABLE). FundsXML 4.x has no XML namespace -> bare element +-- names. Full column list: ../ddl/schema.sql; the runnable Python/Java/ +-- JavaScript/C# programs implement the same mapping end to end. CREATE TABLE fundsxml_stage ( document_id VARCHAR2(128) PRIMARY KEY, doc XMLTYPE NOT NULL ); --- Load (SQL*Plus / SQLcl): use a BFILE or :bind variable, e.g. --- INSERT INTO fundsxml_stage(document_id, doc) --- VALUES ( :doc.extract('/FundsXML4/ControlData/UniqueDocumentID/text()') --- .getStringVal(), XMLTYPE(:doc) ); +INSERT INTO document +SELECT s.document_id, c.generated, c.version, + TO_DATE(c.content_date,'YYYY-MM-DD'), c.data_operation, + c.supplier_country, c.supplier_short, c.supplier_name, c.supplier_type +FROM fundsxml_stage s, + XMLTABLE('/FundsXML4/ControlData' PASSING s.doc COLUMNS + generated VARCHAR2(32) PATH 'DocumentGenerated', + version VARCHAR2(16) PATH 'Version', + content_date VARCHAR2(10) PATH 'ContentDate', + data_operation VARCHAR2(16) PATH 'DataOperation', + supplier_country VARCHAR2(2) PATH 'DataSupplier/SystemCountry', + supplier_short VARCHAR2(64) PATH 'DataSupplier/Short', + supplier_name VARCHAR2(256) PATH 'DataSupplier/Name', + supplier_type VARCHAR2(64) PATH 'DataSupplier/Type') c; -INSERT INTO fund (document_id, lei, official_name, currency, - content_date, nav_date, total_nav, fxml_version) -SELECT s.document_id, f.lei, f.official_name, f.currency, - TO_DATE(f.content_date,'YYYY-MM-DD'), - TO_DATE(f.nav_date,'YYYY-MM-DD'), f.total_nav, f.fxml_version +INSERT INTO fund +SELECT s.document_id, f.fund_seq, f.lei, f.official_name, f.currency, + f.single_fund_flag, TO_DATE(f.nav_date,'YYYY-MM-DD'), f.total_nav FROM fundsxml_stage s, - XMLTABLE('/FundsXML4' PASSING s.doc COLUMNS - lei VARCHAR2(20) PATH 'Funds/Fund/Identifiers/LEI', - official_name VARCHAR2(256) PATH 'Funds/Fund/Names/OfficialName', - currency VARCHAR2(3) PATH 'Funds/Fund/Currency', - content_date VARCHAR2(10) PATH 'ControlData/ContentDate', - fxml_version VARCHAR2(16) PATH 'ControlData/Version', - nav_date VARCHAR2(10) PATH 'Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/NavDate', - total_nav NUMBER PATH 'Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount[@ccy=/FundsXML4/Funds/Fund/Currency]' - ) f; + XMLTABLE('/FundsXML4/Funds/Fund' PASSING s.doc COLUMNS + fund_seq FOR ORDINALITY, + lei VARCHAR2(20) PATH 'Identifiers/LEI', + official_name VARCHAR2(256) PATH 'Names/OfficialName', + currency VARCHAR2(3) PATH 'Currency', + single_fund_flag VARCHAR2(8) PATH 'SingleFundFlag', + nav_date VARCHAR2(10) PATH 'FundDynamicData/TotalAssetValues/TotalAssetValue/NavDate', + total_nav NUMBER PATH 'FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount[@ccy=ancestor::Fund/Currency]') f; -INSERT INTO position (document_id, unique_id, isin, currency, - value_fund_ccy, percentage, kind, kind_qty) -SELECT s.document_id, p.unique_id, p.isin, p.currency, - p.value_fund_ccy, p.percentage, p.kind, p.kind_qty +-- Positions: outer XMLTABLE over Fund (fund_seq + the Fund node), middle over +-- Portfolio (portfolio_seq + node), inner over Position (position_seq). +INSERT INTO position +SELECT s.document_id, fu.fund_seq, pf.portfolio_seq, pos.position_seq, + pos.unique_id, pos.isin, pos.currency, pos.value_fund_ccy, + pos.percentage, pos.kind, pos.kind_qty FROM fundsxml_stage s, - XMLTABLE('/FundsXML4/Funds/Fund/FundDynamicData/Portfolios/Portfolio/Positions/Position' - PASSING s.doc COLUMNS + XMLTABLE('/FundsXML4/Funds/Fund' PASSING s.doc + COLUMNS fund_seq FOR ORDINALITY, + fund_node XMLTYPE PATH '.') fu, + XMLTABLE('FundDynamicData/Portfolios/Portfolio' PASSING fu.fund_node + COLUMNS portfolio_seq FOR ORDINALITY, + port_node XMLTYPE PATH '.') pf, + XMLTABLE('Positions/Position' PASSING pf.port_node COLUMNS + position_seq FOR ORDINALITY, unique_id VARCHAR2(256) PATH 'UniqueID', isin VARCHAR2(12) PATH 'Identifiers/ISIN', currency VARCHAR2(3) PATH 'Currency', value_fund_ccy NUMBER PATH 'TotalValue/Amount[1]', percentage NUMBER PATH 'TotalPercentage', - kind VARCHAR2(16) PATH 'name(*[local-name()=("Equity","Bond","ShareClass","Warrant","Certificate","Option","Future","FXForward","Swap","Repo","RealEstate","CallMoney")][1])', - kind_qty NUMBER PATH '(Equity/Units|Bond/Nominal|ShareClass/Shares|Warrant/Units|Certificate/Units|Option/Contracts|Future/Contracts)[1]' - ) p; + kind VARCHAR2(16) PATH 'name((Equity|Bond|ShareClass|Warrant|Certificate|Option|Future|FXForward|Swap|Repo|RealEstate|CallMoney)[1])', + kind_qty NUMBER PATH '(Equity/Units|Bond/Nominal|ShareClass/Shares|Warrant/Units|Certificate/Units|Option/Contracts|Future/Contracts)[1]') pos; --- asset / share_class: same XMLTABLE pattern over their node sets. +-- portfolio, share_class (per fund) and asset (document-scoped) follow the +-- same nested-XMLTABLE pattern. diff --git a/Database_Integration/load_from_fundsxml/postgres.sql b/Database_Integration/load_from_fundsxml/postgres.sql index efb6d63..18a073c 100644 --- a/Database_Integration/load_from_fundsxml/postgres.sql +++ b/Database_Integration/load_from_fundsxml/postgres.sql @@ -1,56 +1,77 @@ --- FundsXML -> PostgreSQL (code reference; no DB is provisioned in this repo). +-- FundsXML -> PostgreSQL, MULTI-FUND (code reference; no DB is provisioned). -- --- Strategy: stage the document in a native `xml` column, then shred with --- XMLTABLE. FundsXML 4.x has no namespace, so XPath uses bare element names. --- Schema: ../ddl/schema.sql (run that first, plus the staging table below). +-- Pattern: stage the document in a native `xml` column, then shred with +-- nested XMLTABLE. The OUTER XMLTABLE emits one row per and uses +-- FOR ORDINALITY to get fund_seq; inner XMLTABLEs (PASSING the fund/portfolio +-- node) do the same for portfolios and positions. FundsXML 4.x has no XML +-- namespace, so XPath uses bare element names. The runnable Python/Java/ +-- JavaScript/C# examples implement this exact mapping end to end; see +-- ../ddl/schema.sql for the full column list. CREATE TABLE IF NOT EXISTS fundsxml_stage ( document_id text PRIMARY KEY, doc xml NOT NULL ); --- Load the file into the stage (psql): --- \set content `cat FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml` --- INSERT INTO fundsxml_stage --- SELECT (xpath('/FundsXML4/ControlData/UniqueDocumentID/text()', --- x.doc))[1]::text, x.doc --- FROM (SELECT :'content'::xml AS doc) x; +-- document (ControlData) ------------------------------------------------------ +INSERT INTO document +SELECT s.document_id, c.* +FROM fundsxml_stage s, + XMLTABLE('/FundsXML4/ControlData' PASSING s.doc COLUMNS + generated text PATH 'DocumentGenerated', + version text PATH 'Version', + content_date date PATH 'ContentDate', + data_operation text PATH 'DataOperation', + supplier_country text PATH 'DataSupplier/SystemCountry', + supplier_short text PATH 'DataSupplier/Short', + supplier_name text PATH 'DataSupplier/Name', + supplier_type text PATH 'DataSupplier/Type') c; + +-- fund (one row per ; fund_seq via FOR ORDINALITY) --------------------- +INSERT INTO fund +SELECT s.document_id, f.fund_seq, f.lei, f.official_name, f.currency, + f.single_fund_flag, f.nav_date, f.total_nav +FROM fundsxml_stage s, + XMLTABLE('/FundsXML4/Funds/Fund' PASSING s.doc COLUMNS + fund_seq FOR ORDINALITY, + lei text PATH 'Identifiers/LEI', + official_name text PATH 'Names/OfficialName', + currency text PATH 'Currency', + single_fund_flag text PATH 'SingleFundFlag', + nav_date date PATH 'FundDynamicData/TotalAssetValues/TotalAssetValue/NavDate', + total_nav numeric(20,2) PATH 'FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount[@ccy=ancestor::Fund/Currency]') f; --- fund ----------------------------------------------------------------------- -INSERT INTO fund (document_id, lei, official_name, currency, - content_date, nav_date, total_nav, fxml_version) -SELECT s.document_id, f.lei, f.official_name, f.currency, - f.content_date, f.nav_date, f.total_nav, f.fxml_version +-- portfolio + position (nested: portfolios per fund, positions per portfolio) +INSERT INTO portfolio +SELECT s.document_id, fu.fund_seq, pf.portfolio_seq, pf.nav_date FROM fundsxml_stage s, - XMLTABLE('/FundsXML4' PASSING s.doc COLUMNS - lei text PATH 'Funds/Fund/Identifiers/LEI', - official_name text PATH 'Funds/Fund/Names/OfficialName', - currency text PATH 'Funds/Fund/Currency', - content_date date PATH 'ControlData/ContentDate', - fxml_version text PATH 'ControlData/Version', - nav_date date PATH 'Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/NavDate', - total_nav numeric(20,2) PATH 'Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount[@ccy=/FundsXML4/Funds/Fund/Currency]' - ) f; + XMLTABLE('/FundsXML4/Funds/Fund' PASSING s.doc COLUMNS + fund_seq FOR ORDINALITY, + fund_node xml PATH '.') fu, + XMLTABLE('FundDynamicData/Portfolios/Portfolio' PASSING fu.fund_node + COLUMNS portfolio_seq FOR ORDINALITY, + nav_date date PATH 'NavDate') pf; --- position ------------------------------------------------------------------- -INSERT INTO position (document_id, unique_id, isin, currency, - value_fund_ccy, percentage, kind, kind_qty) -SELECT s.document_id, p.unique_id, p.isin, p.currency, - p.value_fund_ccy, p.percentage, p.kind, p.kind_qty +INSERT INTO position +SELECT s.document_id, fu.fund_seq, pf.portfolio_seq, pos.position_seq, + pos.unique_id, pos.isin, pos.currency, pos.value_fund_ccy, + pos.percentage, pos.kind, pos.kind_qty FROM fundsxml_stage s, - LATERAL (SELECT (xpath('/FundsXML4/Funds/Fund/Currency/text()', - s.doc))[1]::text AS ccy) c, - XMLTABLE('/FundsXML4/Funds/Fund/FundDynamicData/Portfolios/Portfolio/Positions/Position' - PASSING s.doc COLUMNS + XMLTABLE('/FundsXML4/Funds/Fund' PASSING s.doc COLUMNS + fund_seq FOR ORDINALITY, ccy text PATH 'Currency', + fund_node xml PATH '.') fu, + XMLTABLE('FundDynamicData/Portfolios/Portfolio' PASSING fu.fund_node + COLUMNS portfolio_seq FOR ORDINALITY, port_node xml PATH '.') pf, + XMLTABLE('Positions/Position' PASSING pf.port_node COLUMNS + position_seq FOR ORDINALITY, unique_id text PATH 'UniqueID', isin text PATH 'Identifiers/ISIN', currency text PATH 'Currency', value_fund_ccy numeric(20,2) PATH 'TotalValue/Amount[1]', percentage numeric(9,4) PATH 'TotalPercentage', kind text PATH 'local-name(Equity|Bond|ShareClass|Warrant|Certificate|Option|Future|FXForward|Swap|Repo|RealEstate|CallMoney)', - kind_qty numeric(28,6) PATH '(Equity/Units|Bond/Nominal|ShareClass/Shares|Warrant/Units|Certificate/Units|Option/Contracts|Future/Contracts)[1]' - ) p; + kind_qty numeric(28,6) PATH '(Equity/Units|Bond/Nominal|ShareClass/Shares|Warrant/Units|Certificate/Units|Option/Contracts|Future/Contracts)[1]') pos; --- asset and share_class follow the same XMLTABLE pattern over --- /FundsXML4/AssetMasterData/Asset and --- /FundsXML4/Funds/Fund/SingleFund/ShareClasses/ShareClass respectively. +-- share_class (per fund) and asset (document-scoped) follow the same nested +-- XMLTABLE pattern over SingleFund/ShareClasses/ShareClass and +-- /FundsXML4/AssetMasterData/Asset respectively. diff --git a/Database_Integration/load_from_fundsxml/sqlserver.sql b/Database_Integration/load_from_fundsxml/sqlserver.sql index 7dc0d40..06b939c 100644 --- a/Database_Integration/load_from_fundsxml/sqlserver.sql +++ b/Database_Integration/load_from_fundsxml/sqlserver.sql @@ -1,47 +1,62 @@ --- FundsXML -> SQL Server (code reference; no DB is provisioned in this repo). +-- FundsXML -> SQL Server, MULTI-FUND (code reference; no DB is provisioned). -- --- Strategy: stage in an `xml` column, shred with .nodes()/.value(). FundsXML --- 4.x has no namespace. Schema: ../ddl/schema.sql. +-- Pattern: stage in an `xml` column; .nodes() shreds the repeating elements. +-- ROW_NUMBER() OVER (ORDER BY (SELECT 1)) over the .nodes() rowset reproduces +-- document order -> fund_seq / portfolio_seq / position_seq. CROSS APPLY +-- chains Fund -> Portfolio -> Position so every fund/portfolio is visited. +-- FundsXML 4.x has no XML namespace. Full columns: ../ddl/schema.sql; the +-- runnable Python/Java/JavaScript/C# programs implement the same mapping. CREATE TABLE fundsxml_stage ( document_id VARCHAR(128) PRIMARY KEY, doc XML NOT NULL ); --- Load: read the file with OPENROWSET(BULK ... SINGLE_CLOB) then --- INSERT INTO fundsxml_stage(document_id, doc) --- SELECT x.value('(/FundsXML4/ControlData/UniqueDocumentID)[1]','varchar(128)'), --- x --- FROM (SELECT CAST(BulkColumn AS XML) AS x --- FROM OPENROWSET(BULK '...Mixed-Fund_Positions.xml', --- SINGLE_CLOB) r) src; - -INSERT INTO fund (document_id, lei, official_name, currency, - content_date, nav_date, total_nav, fxml_version) +INSERT INTO document SELECT s.document_id, - d.value('(Funds/Fund/Identifiers/LEI)[1]','varchar(20)'), - d.value('(Funds/Fund/Names/OfficialName)[1]','varchar(256)'), - d.value('(Funds/Fund/Currency)[1]','char(3)'), - d.value('(ControlData/ContentDate)[1]','date'), - d.value('(Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/NavDate)[1]','date'), - d.value('(Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount)[1]','decimal(20,2)'), - d.value('(ControlData/Version)[1]','varchar(16)') + d.value('(DocumentGenerated)[1]','varchar(32)'), + d.value('(Version)[1]','varchar(16)'), + d.value('(ContentDate)[1]','date'), + d.value('(DataOperation)[1]','varchar(16)'), + d.value('(DataSupplier/SystemCountry)[1]','char(2)'), + d.value('(DataSupplier/Short)[1]','varchar(64)'), + d.value('(DataSupplier/Name)[1]','varchar(256)'), + d.value('(DataSupplier/Type)[1]','varchar(64)') FROM fundsxml_stage s -CROSS APPLY s.doc.nodes('/FundsXML4') AS t(d); +CROSS APPLY s.doc.nodes('/FundsXML4/ControlData') AS t(d); -INSERT INTO position (document_id, unique_id, isin, currency, - value_fund_ccy, percentage, kind, kind_qty) +-- fund_seq from the position of each in document order. +INSERT INTO fund SELECT s.document_id, - p.value('(UniqueID)[1]','varchar(256)'), - p.value('(Identifiers/ISIN)[1]','char(12)'), - p.value('(Currency)[1]','char(3)'), - p.value('(TotalValue/Amount)[1]','decimal(20,2)'), - p.value('(TotalPercentage)[1]','decimal(9,4)'), - p.value('local-name((Equity|Bond|ShareClass|Warrant|Certificate|Option|Future|FXForward|Swap|Repo|RealEstate|CallMoney)[1])','varchar(16)'), - p.value('(Equity/Units|Bond/Nominal|ShareClass/Shares|Warrant/Units|Certificate/Units|Option/Contracts|Future/Contracts)[1]','decimal(28,6)') + ROW_NUMBER() OVER (ORDER BY (SELECT 1)) AS fund_seq, + f.value('(Identifiers/LEI)[1]','varchar(20)'), + f.value('(Names/OfficialName)[1]','varchar(256)'), + f.value('(Currency)[1]','char(3)'), + f.value('(SingleFundFlag)[1]','varchar(8)'), + f.value('(FundDynamicData/TotalAssetValues/TotalAssetValue/NavDate)[1]','date'), + f.value('(FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount)[1]','decimal(20,2)') +FROM fundsxml_stage s +CROSS APPLY s.doc.nodes('/FundsXML4/Funds/Fund') AS t(f); + +-- Positions: number funds, then portfolios within each fund, then positions +-- within each portfolio (CROSS APPLY carries the parent node down). +INSERT INTO position +SELECT s.document_id, fx.fund_seq, px.portfolio_seq, + ROW_NUMBER() OVER (PARTITION BY fx.fund_seq, px.portfolio_seq + ORDER BY (SELECT 1)) AS position_seq, + q.value('(UniqueID)[1]','varchar(256)'), + q.value('(Identifiers/ISIN)[1]','char(12)'), + q.value('(Currency)[1]','char(3)'), + q.value('(TotalValue/Amount)[1]','decimal(20,2)'), + q.value('(TotalPercentage)[1]','decimal(9,4)'), + q.value('local-name((Equity|Bond|ShareClass|Warrant|Certificate|Option|Future|FXForward|Swap|Repo|RealEstate|CallMoney)[1])','varchar(16)'), + q.value('(Equity/Units|Bond/Nominal|ShareClass/Shares|Warrant/Units|Certificate/Units|Option/Contracts|Future/Contracts)[1]','decimal(28,6)') FROM fundsxml_stage s -CROSS APPLY s.doc.nodes( - '/FundsXML4/Funds/Fund/FundDynamicData/Portfolios/Portfolio/Positions/Position' -) AS t(p); +CROSS APPLY (SELECT f.f, ROW_NUMBER() OVER (ORDER BY (SELECT 1)) fund_seq + FROM s.doc.nodes('/FundsXML4/Funds/Fund') AS f(f)) fx +CROSS APPLY (SELECT p.p, ROW_NUMBER() OVER (ORDER BY (SELECT 1)) portfolio_seq + FROM fx.f.nodes('FundDynamicData/Portfolios/Portfolio') AS p(p)) px +CROSS APPLY px.p.nodes('Positions/Position') AS t(q); --- asset / share_class: same CROSS APPLY .nodes() / .value() pattern. +-- portfolio, share_class (per fund) and asset (document-scoped) follow the +-- same numbered CROSS APPLY pattern. diff --git a/Database_Integration/python/fundsxml_db.py b/Database_Integration/python/fundsxml_db.py index 6e46c2d..6f0bb49 100644 --- a/Database_Integration/python/fundsxml_db.py +++ b/Database_Integration/python/fundsxml_db.py @@ -1,119 +1,90 @@ #!/usr/bin/env python3 -"""FundsXML <-> relational database — runnable reference (SQLite, stdlib). - -The Oracle / SQL Server / Postgres examples in ../load_from_fundsxml/ and -../generate_fundsxml/ are code-only (no DB is provisioned, per project scope). -This SQLite implementation is the *executable* reference: it needs nothing -beyond the Python stdlib + lxml, runs the full FundsXML -> DB -> FundsXML -round-trip, and the regenerated file is XSD-valid. - -Subcommands: - init create the schema (ddl/schema.sql) - load shred a FundsXML file into the tables - generate rebuild XSD-valid FundsXML from the tables - roundtrip load + generate via a temp DB (one shot) - -Relational model: ../ddl/schema.sql. FundsXML 4.x has no XML namespace, so all -XPath uses bare element names. -""" +# ============================================================================= +# FundsXML <-> relational database — runnable reference (Python 3, SQLite). +# +# PURPOSE +# A standalone, copy-me example showing BOTH directions: +# * import : read a FundsXML file -> rows in the relational schema +# * export : read those rows -> a FundsXML file +# It is deliberately over-commented: read it top to bottom and you can +# reimplement the same pattern in any language/RDBMS. +# +# DB SCHEMA +# ../ddl/schema.sql (document -> fund -> portfolio -> position; share_class +# per fund; asset document-scoped). SQLite is used so the example runs with +# zero setup; the SQL is plain enough to move to Postgres/Oracle/SQL Server. +# +# RUN +# python3 fundsxml_db.py init fx.db +# python3 fundsxml_db.py import fx.db ../../FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml +# python3 fundsxml_db.py export fx.db FUNDSXML_MULTI_1 out.xml +# python3 fundsxml_db.py roundtrip ../../FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml out.xml +# +# DEPENDENCIES +# Python stdlib `sqlite3` + `lxml` (already used elsewhere in this repo). +# +# FUNDSXML ASSUMPTIONS (important and non-obvious) +# * FundsXML 4.x has NO XML namespace -> XPath uses bare element names. +# * A document may contain MANY , each MANY , each MANY +# . We iterate all of them (no [0] shortcuts) and remember their +# order with 1-based *_seq columns so the export is byte-faithful. +# * Positions link to AssetMasterData by a shared (FundsXML's +# own join key), so `asset` is document-scoped, not per-fund. +# * The export is normalized to the 4.2.9 schema URL; round-trip equality is +# checked with ../tools/xml_equiv.py, which ignores the volatile +# ControlData/DocumentGenerated timestamp and numeric/whitespace +# formatting (see that file for the exact rules). +# +# SECURITY +# The XML parser disables network access and entity resolution (XXE / billion +# laughs). FundsXML never needs DTDs or external entities. +# ============================================================================= import sqlite3 import sys import tempfile +from datetime import datetime, timezone from pathlib import Path from lxml import etree -# Position class elements (the required child after TotalPercentage). Many are -# valid when empty (FXForward/Swap/Repo/RealEstate/CallMoney in the canonical -# sample); the rest accept an empty element for this round-trip demo. +DDL = Path(__file__).resolve().parents[1] / "ddl" / "schema.sql" + +# The official released schema for the version we normalize exports to. Set as +# xsi:noNamespaceSchemaLocation so any validator can find the XSD. +SCHEMA_URL = ("https://github.com/fundsxml/schema/releases/download/" + "4.2.9/FundsXML.xsd") +XSI = "http://www.w3.org/2001/XMLSchema-instance" + +# The "class of instrument" element a must carry (FundsXML models +# this as a choice). We store its name in position.kind and, for the kinds +# that require a quantity child, the numeric quantity in position.kind_qty. +# WHY: without this we could not regenerate a schema-valid Position (an empty +# is invalid — it needs ). Kinds not in QTY_ELEM +# (FXForward/Swap/Repo/RealEstate/CallMoney/...) are valid as an empty element. POSITION_KINDS = {"Equity", "Bond", "ShareClass", "Warrant", "Certificate", "Option", "Future", "FXForward", "Swap", "Repo", "RealEstate", "CallMoney", "Account", "Generic"} -# Kinds that require a numeric quantity child -> the child element name. -# The remaining kinds (FXForward/Swap/Repo/RealEstate/CallMoney/...) are -# schema-valid as an empty element. QTY_ELEM = {"Equity": "Units", "Warrant": "Units", "Certificate": "Units", "Bond": "Nominal", "ShareClass": "Shares", "Option": "Contracts", "Future": "Contracts"} -DDL = Path(__file__).resolve().parents[1] / "ddl" / "schema.sql" -SCHEMA_URL = ("https://github.com/fundsxml/schema/releases/download/" - "4.2.9/FundsXML.xsd") -XSI = "http://www.w3.org/2001/XMLSchema-instance" - - -# ---------------------------------------------------------------- load ------- -def _txt(node, path, default=None): - r = node.xpath(path) - return r[0].text if r and r[0].text is not None else default - - -def load(db: str, xml_path: str) -> str: - doc = etree.parse(xml_path) - cd = doc.xpath("/FundsXML4/ControlData")[0] - fund = doc.xpath("/FundsXML4/Funds/Fund")[0] - doc_id = _txt(cd, "UniqueDocumentID") - ccy = _txt(fund, "Currency") - tav = fund.xpath("FundDynamicData/TotalAssetValues/TotalAssetValue")[0] - nav = tav.xpath(f"TotalNetAssetValue/Amount[@ccy='{ccy}']")[0].text - - con = sqlite3.connect(db) - con.execute("PRAGMA foreign_keys=ON") - con.execute( - "INSERT INTO fund VALUES (?,?,?,?,?,?,?,?)", - (doc_id, _txt(fund, "Identifiers/LEI"), - _txt(fund, "Names/OfficialName"), ccy, - _txt(cd, "ContentDate"), _txt(tav, "NavDate"), - float(nav), _txt(cd, "Version"))) - - for sc in fund.xpath("SingleFund/ShareClasses/ShareClass"): - sccy = _txt(sc, "Currency") - con.execute( - "INSERT INTO share_class VALUES (?,?,?,?,?,?,?)", - (doc_id, _txt(sc, "Identifiers/ISIN"), - _txt(sc, "Names/OfficialName"), sccy, - _num(_txt(sc, "Prices/Price/NavPrice")), - _num(_first(sc, "TotalAssetValues/TotalAssetValue/" - f"TotalNetAssetValue/Amount[@ccy='{ccy}']")), - _num(_txt(sc, "TotalAssetValues/TotalAssetValue/" - "SharesOutstanding")))) - - for a in doc.xpath("/FundsXML4/AssetMasterData/Asset"): - con.execute( - "INSERT INTO asset VALUES (?,?,?,?,?,?,?)", - (doc_id, _txt(a, "UniqueID"), _txt(a, "Identifiers/ISIN"), - _txt(a, "Name"), _txt(a, "AssetType"), - _txt(a, "Currency"), _txt(a, "Country"))) - - for p in fund.xpath("FundDynamicData/Portfolios/Portfolio/Positions/Position"): - kinds = [c.tag for c in p if c.tag in POSITION_KINDS] - kind = kinds[0] if kinds else None - qty = None - if kind in QTY_ELEM: - qty = _num(_txt(p, f"{kind}/{QTY_ELEM[kind]}")) - con.execute( - "INSERT INTO position VALUES (?,?,?,?,?,?,?,?)", - (doc_id, _txt(p, "UniqueID"), _txt(p, "Identifiers/ISIN"), - _txt(p, "Currency"), - float(p.xpath(f"TotalValue/Amount[@ccy='{ccy}']")[0].text), - float(_txt(p, "TotalPercentage")), - kind, qty)) - con.commit() - con.close() - return doc_id - -def _first(node, path): +# --------------------------------------------------------------------------- +# small helpers +# --------------------------------------------------------------------------- +def _text(node, path): + """First matching element's text, or None. Bare names (no namespace).""" r = node.xpath(path) - return r[0].text if r else None + return r[0].text if r and r[0].text is not None else None def _num(v): return float(v) if v not in (None, "") else None -# ------------------------------------------------------------ generate ------- def _el(parent, tag, text=None, **attrs): + """Append a child element; the one XML-building primitive used throughout + the exporter so the emitted structure stays consistent and reviewable.""" e = etree.SubElement(parent, tag) for k, v in attrs.items(): e.set(k, str(v)) @@ -122,97 +93,222 @@ def _el(parent, tag, text=None, **attrs): return e -def generate(db: str, document_id: str, out: str) -> None: +def connect(db): con = sqlite3.connect(db) + con.execute("PRAGMA foreign_keys = ON") # enforce the FK graph con.row_factory = sqlite3.Row - f = con.execute("SELECT * FROM fund WHERE document_id=?", + return con + + +# --------------------------------------------------------------------------- +# init : create the schema +# --------------------------------------------------------------------------- +def init(db): + con = connect(db) + con.executescript(DDL.read_text()) + con.commit() + con.close() + + +# --------------------------------------------------------------------------- +# import : FundsXML -> relational rows +# +# We walk the document in its natural order and assign 1-based ordinals +# (fund_seq / portfolio_seq / position_seq). Those ordinals are what make the +# export deterministic and therefore comparable to the original file. +# --------------------------------------------------------------------------- +def do_import(db, xml_path): + parser = etree.XMLParser(resolve_entities=False, no_network=True, + load_dtd=False, huge_tree=False) + doc = etree.parse(xml_path, parser) + cd = doc.xpath("/FundsXML4/ControlData")[0] + document_id = _text(cd, "UniqueDocumentID") + + con = connect(db) + + # ControlData -> one `document` row (so it round-trips exactly instead of + # being hard-coded by the exporter). + con.execute( + "INSERT INTO document VALUES (?,?,?,?,?,?,?,?,?)", + (document_id, _text(cd, "DocumentGenerated"), _text(cd, "Version"), + _text(cd, "ContentDate"), _text(cd, "DataOperation"), + _text(cd, "DataSupplier/SystemCountry"), + _text(cd, "DataSupplier/Short"), + _text(cd, "DataSupplier/Name"), + _text(cd, "DataSupplier/Type"))) + + # Every (not just the first). + for fund_seq, fund in enumerate(doc.xpath("/FundsXML4/Funds/Fund"), start=1): + ccy = _text(fund, "Currency") + tav = fund.xpath("FundDynamicData/TotalAssetValues/TotalAssetValue")[0] + con.execute( + "INSERT INTO fund VALUES (?,?,?,?,?,?,?,?)", + (document_id, fund_seq, _text(fund, "Identifiers/LEI"), + _text(fund, "Names/OfficialName"), ccy, + _text(fund, "SingleFundFlag"), _text(tav, "NavDate"), + float(tav.xpath( + f"TotalNetAssetValue/Amount[@ccy='{ccy}']")[0].text))) + + # Share classes of THIS fund. + for sc in fund.xpath("SingleFund/ShareClasses/ShareClass"): + navf = sc.xpath("TotalAssetValues/TotalAssetValue/" + f"TotalNetAssetValue/Amount[@ccy='{ccy}']") + con.execute( + "INSERT INTO share_class VALUES (?,?,?,?,?,?,?,?)", + (document_id, fund_seq, _text(sc, "Identifiers/ISIN"), + _text(sc, "Names/OfficialName"), _text(sc, "Currency"), + _num(_text(sc, "Prices/Price/NavPrice")), + _num(navf[0].text) if navf else None, + _num(_text(sc, "TotalAssetValues/TotalAssetValue/" + "SharesOutstanding")))) + + # Every of this fund, and every in it. + for p_seq, port in enumerate( + fund.xpath("FundDynamicData/Portfolios/Portfolio"), start=1): + con.execute("INSERT INTO portfolio VALUES (?,?,?,?)", + (document_id, fund_seq, p_seq, _text(port, "NavDate"))) + for pos_seq, pos in enumerate( + port.xpath("Positions/Position"), start=1): + # The Position's instrument-class child (Equity/Bond/...). + kinds = [c.tag for c in pos if c.tag in POSITION_KINDS] + kind = kinds[0] if kinds else None + qty = (_num(_text(pos, f"{kind}/{QTY_ELEM[kind]}")) + if kind in QTY_ELEM else None) + con.execute( + "INSERT INTO position VALUES (?,?,?,?,?,?,?,?,?,?,?)", + (document_id, fund_seq, p_seq, pos_seq, + _text(pos, "UniqueID"), _text(pos, "Identifiers/ISIN"), + _text(pos, "Currency"), + float(pos.xpath( + f"TotalValue/Amount[@ccy='{ccy}']")[0].text), + float(_text(pos, "TotalPercentage")), kind, qty)) + + # AssetMasterData is shared by all funds (document-scoped). + for a in doc.xpath("/FundsXML4/AssetMasterData/Asset"): + con.execute( + "INSERT INTO asset VALUES (?,?,?,?,?,?,?)", + (document_id, _text(a, "UniqueID"), _text(a, "Identifiers/ISIN"), + _text(a, "Name"), _text(a, "AssetType"), + _text(a, "Currency"), _text(a, "Country"))) + + con.commit() + con.close() + return document_id + + +# --------------------------------------------------------------------------- +# export : relational rows -> FundsXML +# +# Emits a fixed, canonical structure in schema order. Constants that the model +# does not store (TotalAssetNature=OFFICIAL, Price ActionCode=C, +# PriceNature=OFFICIAL) are reproduced verbatim; the round-trip fixture uses +# the same constants, so xml_equiv.py sees the files as equal. +# --------------------------------------------------------------------------- +def export(db, document_id, out_path): + con = connect(db) + d = con.execute("SELECT * FROM document WHERE document_id=?", (document_id,)).fetchone() - if f is None: - raise SystemExit(f"no fund with document_id {document_id!r}") - ccy = f["currency"] + if d is None: + raise SystemExit(f"no document {document_id!r} in {db}") root = etree.Element("FundsXML4", nsmap={"xsi": XSI}) root.set(f"{{{XSI}}}noNamespaceSchemaLocation", SCHEMA_URL) cd = _el(root, "ControlData") - _el(cd, "UniqueDocumentID", f["document_id"]) - _el(cd, "DocumentGenerated", "2025-10-02T00:00:00") - if f["fxml_version"]: - _el(cd, "Version", f["fxml_version"]) - _el(cd, "ContentDate", f["content_date"]) + _el(cd, "UniqueDocumentID", d["document_id"]) + # Regenerate the timestamp; xml_equiv.py ignores its value by design. + _el(cd, "DocumentGenerated", + d["generated"] or datetime.now(timezone.utc) + .strftime("%Y-%m-%dT%H:%M:%S")) + if d["version"]: # absent for FundsXML 4.0.0 + _el(cd, "Version", d["version"]) + _el(cd, "ContentDate", d["content_date"]) ds = _el(cd, "DataSupplier") - _el(ds, "SystemCountry", "AT") - _el(ds, "Short", "EURAM") - _el(ds, "Name", "Erste Asset Management GmbH") - _el(ds, "Type", "Asset Manager") - _el(cd, "DataOperation", "INITIAL") - - fund = _el(_el(root, "Funds"), "Fund") - if f["lei"]: - _el(_el(fund, "Identifiers"), "LEI", f["lei"]) - _el(_el(fund, "Names"), "OfficialName", f["official_name"]) - _el(fund, "Currency", ccy) - _el(fund, "SingleFundFlag", "true") - - fdd = _el(fund, "FundDynamicData") - tav = _el(_el(_el(fdd, "TotalAssetValues"), "TotalAssetValue"), "NavDate", - f["nav_date"]).getparent() - _el(tav, "TotalAssetNature", "OFFICIAL") - _el(_el(tav, "TotalNetAssetValue"), "Amount", - f'{f["total_nav"]:.2f}', ccy=ccy) - - port = _el(_el(fdd, "Portfolios"), "Portfolio") - _el(port, "NavDate", f["nav_date"]) - poss = _el(port, "Positions") - for p in con.execute( - "SELECT * FROM position WHERE document_id=? ORDER BY unique_id", - (document_id,)): - pos = _el(poss, "Position") - _el(pos, "UniqueID", p["unique_id"]) - if p["isin"]: - _el(_el(pos, "Identifiers"), "ISIN", p["isin"]) - if p["currency"]: - _el(pos, "Currency", p["currency"]) - _el(_el(pos, "TotalValue"), "Amount", - f'{p["value_fund_ccy"]:.2f}', ccy=ccy) - _el(pos, "TotalPercentage", f'{p["percentage"]:.2f}') - # Required Position class element. Kinds with a mandatory quantity - # child get it back (Units/Nominal/Shares/Contracts); the rest are - # schema-valid empty. - kind = p["kind"] if p["kind"] in POSITION_KINDS else "Generic" - ke = _el(pos, kind) - if kind in QTY_ELEM and p["kind_qty"] is not None: - _el(ke, QTY_ELEM[kind], f'{p["kind_qty"]:.2f}') - - scs = con.execute("SELECT * FROM share_class WHERE document_id=? " - "ORDER BY isin", (document_id,)).fetchall() - if scs: - sce = _el(_el(fund, "SingleFund"), "ShareClasses") - for sc in scs: - x = _el(sce, "ShareClass") - _el(_el(x, "Identifiers"), "ISIN", sc["isin"]) - if sc["official_name"]: - _el(_el(x, "Names"), "OfficialName", sc["official_name"]) - _el(x, "Currency", sc["currency"]) - if sc["nav_price"] is not None: - pr = _el(_el(x, "Prices"), "Price") - _el(pr, "ActionCode", "C") - _el(pr, "NavDate", f["nav_date"]) - _el(pr, "PriceCurrency", sc["currency"]) - _el(pr, "PriceNature", "OFFICIAL") - _el(pr, "NavPrice", f'{sc["nav_price"]:.2f}') - if sc["nav_fund_ccy"] is not None: - t = _el(_el(x, "TotalAssetValues"), "TotalAssetValue") - _el(t, "NavDate", f["nav_date"]) - _el(t, "TotalAssetNature", "OFFICIAL") - _el(_el(t, "TotalNetAssetValue"), "Amount", - f'{sc["nav_fund_ccy"]:.2f}', ccy=ccy) - if sc["shares_outstanding"] is not None: - _el(t, "SharesOutstanding", - f'{sc["shares_outstanding"]:.0f}') - - assets = con.execute("SELECT * FROM asset WHERE document_id=? " - "ORDER BY unique_id", (document_id,)).fetchall() + _el(ds, "SystemCountry", d["supplier_country"]) + _el(ds, "Short", d["supplier_short"]) + _el(ds, "Name", d["supplier_name"]) + _el(ds, "Type", d["supplier_type"]) + _el(cd, "DataOperation", d["data_operation"]) + + funds = con.execute( + "SELECT * FROM fund WHERE document_id=? ORDER BY fund_seq", + (document_id,)).fetchall() + funds_el = _el(root, "Funds") + for f in funds: + ccy = f["currency"] + fund = _el(funds_el, "Fund") + if f["lei"]: + _el(_el(fund, "Identifiers"), "LEI", f["lei"]) + _el(_el(fund, "Names"), "OfficialName", f["official_name"]) + _el(fund, "Currency", ccy) + if f["single_fund_flag"]: + _el(fund, "SingleFundFlag", f["single_fund_flag"]) + + fdd = _el(fund, "FundDynamicData") + tav = _el(_el(_el(fdd, "TotalAssetValues"), "TotalAssetValue"), + "NavDate", f["nav_date"]).getparent() + _el(tav, "TotalAssetNature", "OFFICIAL") + _el(_el(tav, "TotalNetAssetValue"), "Amount", + f'{f["total_nav"]:.2f}', ccy=ccy) + + ports = _el(fdd, "Portfolios") + for pf in con.execute( + "SELECT * FROM portfolio WHERE document_id=? AND fund_seq=? " + "ORDER BY portfolio_seq", (document_id, f["fund_seq"])): + pe = _el(ports, "Portfolio") + _el(pe, "NavDate", pf["nav_date"]) + poss = _el(pe, "Positions") + for p in con.execute( + "SELECT * FROM position WHERE document_id=? AND fund_seq=? " + "AND portfolio_seq=? ORDER BY position_seq", + (document_id, f["fund_seq"], pf["portfolio_seq"])): + pos = _el(poss, "Position") + _el(pos, "UniqueID", p["unique_id"]) + if p["isin"]: + _el(_el(pos, "Identifiers"), "ISIN", p["isin"]) + if p["currency"]: + _el(pos, "Currency", p["currency"]) + _el(_el(pos, "TotalValue"), "Amount", + f'{p["value_fund_ccy"]:.2f}', ccy=ccy) + _el(pos, "TotalPercentage", f'{p["percentage"]:.2f}') + kind = p["kind"] if p["kind"] in POSITION_KINDS else "Generic" + ke = _el(pos, kind) + if kind in QTY_ELEM and p["kind_qty"] is not None: + _el(ke, QTY_ELEM[kind], f'{p["kind_qty"]:.2f}') + + # Share classes (authored in ISIN order in the fixture, so ORDER BY + # isin makes export child-order match the original). + scs = con.execute( + "SELECT * FROM share_class WHERE document_id=? AND fund_seq=? " + "ORDER BY isin", (document_id, f["fund_seq"])).fetchall() + if scs: + sce = _el(_el(fund, "SingleFund"), "ShareClasses") + for sc in scs: + x = _el(sce, "ShareClass") + _el(_el(x, "Identifiers"), "ISIN", sc["isin"]) + if sc["official_name"]: + _el(_el(x, "Names"), "OfficialName", sc["official_name"]) + _el(x, "Currency", sc["currency"]) + if sc["nav_price"] is not None: + pr = _el(_el(x, "Prices"), "Price") + _el(pr, "ActionCode", "C") + _el(pr, "NavDate", f["nav_date"]) + _el(pr, "PriceCurrency", sc["currency"]) + _el(pr, "PriceNature", "OFFICIAL") + _el(pr, "NavPrice", f'{sc["nav_price"]:.2f}') + if sc["nav_fund_ccy"] is not None: + t = _el(_el(x, "TotalAssetValues"), "TotalAssetValue") + _el(t, "NavDate", f["nav_date"]) + _el(t, "TotalAssetNature", "OFFICIAL") + _el(_el(t, "TotalNetAssetValue"), "Amount", + f'{sc["nav_fund_ccy"]:.2f}', ccy=ccy) + if sc["shares_outstanding"] is not None: + _el(t, "SharesOutstanding", + f'{sc["shares_outstanding"]:.0f}') + + assets = con.execute( + "SELECT * FROM asset WHERE document_id=? ORDER BY unique_id", + (document_id,)).fetchall() if assets: amd = _el(root, "AssetMasterData") for a in assets: @@ -220,46 +316,44 @@ def generate(db: str, document_id: str, out: str) -> None: _el(ae, "UniqueID", a["unique_id"]) if a["isin"]: _el(_el(ae, "Identifiers"), "ISIN", a["isin"]) - _el(ae, "Currency", a["currency"] or ccy) + _el(ae, "Currency", a["currency"]) if a["country"]: _el(ae, "Country", a["country"]) _el(ae, "Name", a["name"]) _el(ae, "AssetType", a["asset_type"]) con.close() - etree.ElementTree(root).write(out, xml_declaration=True, + etree.ElementTree(root).write(out_path, xml_declaration=True, encoding="UTF-8", pretty_print=True) -# ---------------------------------------------------------------- cli -------- -def _init(db: str) -> None: - con = sqlite3.connect(db) - con.executescript(DDL.read_text()) - con.commit() - con.close() - - +# --------------------------------------------------------------------------- +# cli +# --------------------------------------------------------------------------- def main() -> int: a = sys.argv[1:] if not a: print(__doc__, file=sys.stderr) return 2 - if a[0] == "init": - _init(a[1]) - elif a[0] == "load": - print("loaded document_id:", load(a[1], a[2])) - elif a[0] == "generate": - generate(a[1], a[2], a[3]) + cmd = a[0] + if cmd == "init": + init(a[1]) + elif cmd == "import": + print("imported document_id:", do_import(a[1], a[2])) + elif cmd == "export": + export(a[1], a[2], a[3]) print("wrote", a[3]) - elif a[0] == "roundtrip": + elif cmd == "roundtrip": + # import then export THROUGH the DB (the user's required test): the + # data exported is exactly the data the import wrote. with tempfile.TemporaryDirectory() as t: - db = str(Path(t) / "fundsxml.db") - _init(db) - doc_id = load(db, a[1]) - generate(db, doc_id, a[2]) + db = str(Path(t) / "rt.db") + init(db) + doc_id = do_import(db, a[1]) + export(db, doc_id, a[2]) print(f"round-trip ok: {a[1]} -> DB -> {a[2]} (doc {doc_id})") else: - print(f"unknown subcommand {a[0]!r}", file=sys.stderr) + print(f"unknown command {cmd!r}", file=sys.stderr) return 2 return 0 diff --git a/Database_Integration/tools/xml_equiv.py b/Database_Integration/tools/xml_equiv.py new file mode 100644 index 0000000..b09ddac --- /dev/null +++ b/Database_Integration/tools/xml_equiv.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Normalized FundsXML equivalence check — proves import file == export file. + +Usage: xml_equiv.py +Exit: 0 = equivalent, 1 = differ (first difference printed), 2 = usage + +WHY this tool exists +-------------------- +The database round-trip examples must demonstrate that exporting the rows the +import wrote yields *the same* FundsXML document. A byte comparison is wrong: +two equivalent FundsXML files legitimately differ in ways that carry no +information. This comparator treats two documents as equal when they differ +ONLY by: + + * ControlData/DocumentGenerated — a generation timestamp; volatile by + design, so its text is ignored (the element must still be present). + * Numeric formatting — 9375000 == 9375000.00 == 9.375E6; + elements/attributes whose values are numbers compare as numbers. + * Insignificant whitespace — pretty-print indentation / text padding. + * Attribute order & xmlns form — attributes compare as a map; namespace + declarations are not data. + +Everything else is significant. FundsXML content models are xs:sequence, so +**child order is compared** (a reordered document is NOT considered equal) — +this is deliberately strict so the round-trip really proves fidelity. + +Shared by the Python/Java/C#/JavaScript examples (via CI) so every language is +held to the identical definition of "round-trips correctly". + +NOTE: namespace declarations are treated as non-data, so this check ALONE +would accept an xsi:* attribute that lost its namespace. Always pair it with +XSD validation (xmllint --schema ...) — the examples and CI always do. +""" +import sys + +from lxml import etree + +# Fully-qualified path (tag chain from the root) whose text is volatile. +IGNORE_TEXT_PATHS = {"FundsXML4/ControlData/DocumentGenerated"} + + +def _norm_num(s): + """Return a canonical form: a float if the string is numeric, else the + whitespace-stripped string. Lets 100 == 100.00 == 1E2 compare equal.""" + s = (s or "").strip() + try: + return ("num", float(s)) + except (TypeError, ValueError): + return ("str", s) + + +def _children(el): + # Element children only (drop comments/PIs); whitespace handled via text. + return [c for c in el if isinstance(c.tag, str)] + + +def diff(a, b, path=""): + """Return None if equal, else a human-readable description of the first + difference (depth-first, document order).""" + here = f"{path}/{etree.QName(a).localname}" if path else etree.QName(a).localname + + if etree.QName(a).localname != etree.QName(b).localname: + return f"{path or '/'}: tag {etree.QName(a).localname} != {etree.QName(b).localname}" + + # Attributes as numeric-aware maps (local names; xmlns not data). + aa = {etree.QName(k).localname: _norm_num(v) for k, v in a.attrib.items()} + ba = {etree.QName(k).localname: _norm_num(v) for k, v in b.attrib.items()} + if aa != ba: + return f"{here}: attributes {aa} != {ba}" + + ac, bc = _children(a), _children(b) + if not ac and not bc: + # Leaf: compare text unless this path is whitelisted as volatile. + if here in IGNORE_TEXT_PATHS: + return None + if _norm_num(a.text) != _norm_num(b.text): + return (f"{here}: text {(a.text or '').strip()!r} != " + f"{(b.text or '').strip()!r}") + return None + + if len(ac) != len(bc): + return (f"{here}: child count {len(ac)} != {len(bc)} " + f"(a=[{','.join(etree.QName(c).localname for c in ac)}] " + f"b=[{','.join(etree.QName(c).localname for c in bc)}])") + for ca, cb in zip(ac, bc): + d = diff(ca, cb, here) + if d: + return d + return None + + +def main() -> int: + if len(sys.argv) != 3: + print("usage: xml_equiv.py ", file=sys.stderr) + return 2 + p = etree.XMLParser(remove_blank_text=True, resolve_entities=False, + no_network=True) + a = etree.parse(sys.argv[1], p).getroot() + b = etree.parse(sys.argv[2], p).getroot() + d = diff(a, b) + if d is None: + print(f"EQUIVALENT: {sys.argv[1]} == {sys.argv[2]} " + f"(ignoring timestamp & numeric/whitespace formatting)") + return 0 + print(f"DIFFER: {d}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml b/FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml new file mode 100644 index 0000000..b9731ff --- /dev/null +++ b/FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml @@ -0,0 +1,276 @@ + + + + + FUNDSXML_MULTI_1 + 2025-10-02T00:00:00 + 4.2.9 + 2025-10-01 + + AT + EURAM + Erste Asset Management GmbH + Asset Manager + + INITIAL + + + + + 529900T8BM49AURSDO55 + + + Erste Multi Fund Alpha + + EUR + false + + + + 2025-10-01 + OFFICIAL + + 25000000.00 + + + + + + 2025-10-01 + + + MF_001 + + US0378331005 + + EUR + + 8000000.00 + + 32.00 + + 50000.00 + + + + MF_002 + + DE0001102424 + + EUR + + 6000000.00 + + 24.00 + + 6000000.00 + + + + + + 2025-10-01 + + + MF_003 + + IE00B5BMR087 + + EUR + + 10000000.00 + + 40.00 + + 20000.00 + + + + MF_004 + EUR + + 1000000.00 + + 4.00 + + + + + + + + + + + AT0000A2QM66 + + + Erste Multi Fund Alpha EUR R01 + + EUR + + + C + 2025-10-01 + EUR + OFFICIAL + 100.00 + + + + + 2025-10-01 + OFFICIAL + + 15000000.00 + + 150000 + + + + + + AT0000A2QM74 + + + Erste Multi Fund Alpha EUR I01 + + EUR + + + C + 2025-10-01 + EUR + OFFICIAL + 100.00 + + + + + 2025-10-01 + OFFICIAL + + 10000000.00 + + 100000 + + + + + + + + + 529900T8BM49AURSDO66 + + + Erste Multi Fund Beta + + EUR + true + + + + 2025-10-01 + OFFICIAL + + 10000000.00 + + + + + + 2025-10-01 + + + MF_101 + + NL0010273215 + + EUR + + 9000000.00 + + 90.00 + + 10000.00 + + + + MF_102 + EUR + + 1000000.00 + + 10.00 + + + + + + + + + + + MF_001 + + US0378331005 + + EUR + US + APPLE INC. + EQ + + + MF_002 + + DE0001102424 + + EUR + DE + Germany, Federal Republic of 1.70% 2032 + BO + + + MF_003 + + IE00B5BMR087 + + EUR + IE + Vanguard S&P 500 UCITS ETF EUR (Acc) + SC + + + MF_004 + EUR + EUR 5Y Interest Rate Swap + SW + + + MF_101 + + NL0010273215 + + EUR + NL + ASML HOLDING N.V. + EQ + + + MF_102 + EUR + FX Forward EUR/USD + FX + + + diff --git a/README.md b/README.md index 3c3cb22..db8c3bb 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ locations. Items marked _(planned)_ are on the roadmap (see | CI (validate all samples) | GitHub Actions | [.github/workflows/ci.yml](./.github/workflows/) | ✅ | | XQuery analytics (aggregation, top-holdings, look-through) | Saxon CLI/Java, Python, BaseX | [XQuery_Examples/](./XQuery_Examples/) | ✅ | | XML signature sign/verify | Apache Santuario (Java), .NET, xmlsec1, signxml | [XML_Signature/](./XML_Signature/) | ✅ | -| Database load ↔ generate | SQLite (verified) + Oracle/SQL Server/Postgres (code) | [Database_Integration/](./Database_Integration/) | ✅ | +| Database load ↔ generate (multi-fund) | Python · Java · JavaScript · C# standalone (SQLite), all verified; Oracle/SQL Server/Postgres SQL (code) | [Database_Integration/](./Database_Integration/) | ✅ | | Large-file / streaming | lxml iterparse + Java StAX, split, delta-diff | [Large_File_Processing/](./Large_File_Processing/) | ✅ | | Data binding & JSON | FundsXML⇄JSON, native Java binding, codegen refs | [Data_Binding_JSON/](./Data_Binding_JSON/) | ✅ | diff --git a/tools/fetch-tools.sh b/tools/fetch-tools.sh index 2e589b9..aae1c91 100755 --- a/tools/fetch-tools.sh +++ b/tools/fetch-tools.sh @@ -52,6 +52,12 @@ fetch "${M2}/commons-codec/commons-codec/1.18.0/commons-codec-1.18.0.jar" "${LIB fetch "${M2}/org/slf4j/slf4j-api/2.0.17/slf4j-api-2.0.17.jar" "${LIB}/slf4j-api-2.0.17.jar" fetch "${M2}/org/slf4j/slf4j-nop/2.0.17/slf4j-nop-2.0.17.jar" "${LIB}/slf4j-nop-2.0.17.jar" +# --- SQLite JDBC (Database_Integration Java example) ------------------------- +# Self-contained driver: bundles the native SQLite libraries for common +# platforms, so the Java example needs no system SQLite install. +fetch "${M2}/org/xerial/sqlite-jdbc/3.46.1.3/sqlite-jdbc-3.46.1.3.jar" "${LIB}/sqlite-jdbc-3.46.1.3.jar" + echo "SAXON_CP=${LIB}/Saxon-HE-12.5.jar:${LIB}/xmlresolver-5.2.2.jar:${LIB}/xmlresolver-5.2.2-data.jar" +echo "SQLITE_JDBC=${LIB}/sqlite-jdbc-3.46.1.3.jar" echo "SCHXSLT_CP=${LIB}/schxslt-cli-1.10.1.jar:${LIB}/commons-cli-1.5.0.jar:${LIB}/slf4j-api-1.7.32.jar:${LIB}/slf4j-nop-1.7.32.jar" echo "SANTUARIO_CP=${LIB}/xmlsec-4.0.4.jar:${LIB}/commons-codec-1.18.0.jar:${LIB}/slf4j-api-2.0.17.jar:${LIB}/slf4j-nop-2.0.17.jar"