From ae48398a21d6e95475d2c93247a38d0441f2a791 Mon Sep 17 00:00:00 2001 From: Karl Kauc Date: Fri, 15 May 2026 18:47:00 +0200 Subject: [PATCH] Phase 2: XQuery analytics + per-stack invocation + CI XQuery_Examples/: aggregate-by-assettype, top-holdings (external var n), fund-summary (version-agnostic + reconciliation), look-through (fund-of-funds readiness). Position<->Asset joined by UniqueID; no XML namespace so they run unchanged on 4.2.9/4.1.0/4.0.0. invocation/: Saxon CLI (run-xquery.sh) and native Java s9api (RunXQuery.java) verified; Python saxonche reference; BaseX documented. README documents Saxon Query's bare name=value external-variable syntax. CI: added an XQuery smoke step (all four queries run, output well-formed, Holding count asserted). Top-level index updated. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 14 ++++++ .gitignore | 1 + README.md | 2 +- XQuery_Examples/README.md | 47 ++++++++++++++++++++ XQuery_Examples/aggregate-by-assettype.xq | 45 +++++++++++++++++++ XQuery_Examples/fund-summary.xq | 54 +++++++++++++++++++++++ XQuery_Examples/invocation/RunXQuery.java | 48 ++++++++++++++++++++ XQuery_Examples/invocation/run-xquery.sh | 32 ++++++++++++++ XQuery_Examples/invocation/run_xquery.py | 39 ++++++++++++++++ XQuery_Examples/look-through.xq | 47 ++++++++++++++++++++ XQuery_Examples/top-holdings.xq | 37 ++++++++++++++++ 11 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 XQuery_Examples/README.md create mode 100644 XQuery_Examples/aggregate-by-assettype.xq create mode 100644 XQuery_Examples/fund-summary.xq create mode 100644 XQuery_Examples/invocation/RunXQuery.java create mode 100755 XQuery_Examples/invocation/run-xquery.sh create mode 100644 XQuery_Examples/invocation/run_xquery.py create mode 100644 XQuery_Examples/look-through.xq create mode 100644 XQuery_Examples/top-holdings.xq diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54defb2..a4b087b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,20 @@ jobs: done test "$(wc -l < out_pos.csv)" -ge 2 + - name: Smoke - XQuery examples run and produce XML + run: | + set -e + RQ=XQuery_Examples/invocation/run-xquery.sh + SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml + "$RQ" XQuery_Examples/aggregate-by-assettype.xq "$SRC" out_agg.xml + "$RQ" XQuery_Examples/top-holdings.xq "$SRC" out_top.xml n=5 + "$RQ" XQuery_Examples/fund-summary.xq "$SRC" out_sum.xml + "$RQ" XQuery_Examples/look-through.xq "$SRC" out_lt.xml + for f in out_agg.xml out_top.xml out_sum.xml out_lt.xml; do + xmllint --noout "$f" + done + test "$(xmllint --xpath 'count(//Holding)' out_top.xml)" = "5" + - name: Regression - legacy XSLT 1.0 report still runs run: | xsltproc XSLT_DataQuality_Checks/Enhanced_Check/FundsXML_CompleteDQReport_HTML.xsl \ diff --git a/.gitignore b/.gitignore index 77c3020..e863448 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ out/ out_*.html out_*.fo out_*.csv +out_*.xml *.svrl diff --git a/README.md b/README.md index ec8bc6e..ecb15bb 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ locations. Items marked _(planned)_ are on the roadmap (see | Transformation invocation | CLI, Python, Java, .NET, Node | [XSLT_Transformations/invocation/](./XSLT_Transformations/) | ✅ | | Schema fetch (proxy-aware) | Bash | [tools/fetch-schema.sh](./tools/fetch-schema.sh) | ✅ | | CI (validate all samples) | GitHub Actions | [.github/workflows/ci.yml](./.github/workflows/) | ✅ | -| XQuery examples | BaseX/Saxon, Python, Java, .NET | `XQuery_Examples/` | _(planned)_ | +| XQuery analytics (aggregation, top-holdings, look-through) | Saxon CLI/Java, Python, BaseX | [XQuery_Examples/](./XQuery_Examples/) | ✅ | | XML signature sign/verify | Apache Santuario, .NET, xmlsec1, signxml | `XML_Signature/` | _(planned)_ | | Database load ↔ generate | Oracle/SQL Server/Postgres (code only) | `Database_Integration/` | _(planned)_ | | Large-file/stream processing | StAX/SAX/lxml iterparse | `Large_File_Processing/` | _(planned)_ | diff --git a/XQuery_Examples/README.md b/XQuery_Examples/README.md new file mode 100644 index 0000000..350803f --- /dev/null +++ b/XQuery_Examples/README.md @@ -0,0 +1,47 @@ +# XQuery Examples + +![XQuery](https://img.shields.io/badge/XQuery-3.1-blue) ![status](https://img.shields.io/badge/status-verified-brightgreen) + +Read-only analytics over FundsXML with XQuery 3.1. Positions and assets are +joined by the shared `UniqueID` (the standard FundsXML link). FundsXML 4.x has +no XML namespace — queries use bare element names, so they work unchanged on the +4.2.9 / 4.1.0 / 4.0.0 samples. + +| Query | Purpose | Notable | +|-------|---------|---------| +| [`aggregate-by-assettype.xq`](aggregate-by-assettype.xq) | Exposure grouped by `AssetType` | `group by`, grand-total reconciliation | +| [`top-holdings.xq`](top-holdings.xq) | N largest holdings, names resolved | external var `n` (default 10) | +| [`fund-summary.xq`](fund-summary.xq) | Fund/doc overview + reconciliation block | version-agnostic (4.0.0 has no `Version`) | +| [`look-through.xq`](look-through.xq) | Fund-of-funds look-through readiness | flags `SC` (fund) positions + weights | + +## Run (per stack) + +| Stack | Entry point | Status | +|-------|-------------|--------| +| CLI (Saxon) | [`invocation/run-xquery.sh`](invocation/run-xquery.sh) | ✅ verified | +| Java (s9api, no JAXB) | [`invocation/RunXQuery.java`](invocation/RunXQuery.java) | ✅ verified | +| Python | [`invocation/run_xquery.py`](invocation/run_xquery.py) | needs `pip install saxonche` | +| BaseX | see below | needs BaseX install | + +```bash +tools/fetch-tools.sh +SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml + +# CLI — to stdout +XQuery_Examples/invocation/run-xquery.sh XQuery_Examples/fund-summary.xq "$SRC" + +# CLI — external variable + output file (Saxon Query: bare name=value) +XQuery_Examples/invocation/run-xquery.sh XQuery_Examples/top-holdings.xq "$SRC" top.xml n=5 + +# Java s9api +SCP=.lib/Saxon-HE-12.5.jar:.lib/xmlresolver-5.2.2.jar:.lib/xmlresolver-5.2.2-data.jar +javac -cp "$SCP" -d /tmp/xq XQuery_Examples/invocation/RunXQuery.java +java -cp "$SCP:/tmp/xq" RunXQuery XQuery_Examples/aggregate-by-assettype.xq "$SRC" + +# BaseX (in-memory, binds the document and the external variable) +basex -i "$SRC" -bn=5 XQuery_Examples/top-holdings.xq +``` + +Note: Saxon's `net.sf.saxon.Query` takes external variables as plain +`name=value` arguments — **not** `-param:` (that is the XSLT entry point) and +not `!name=value` (that sets serialization properties). diff --git a/XQuery_Examples/aggregate-by-assettype.xq b/XQuery_Examples/aggregate-by-assettype.xq new file mode 100644 index 0000000..dcd0e08 --- /dev/null +++ b/XQuery_Examples/aggregate-by-assettype.xq @@ -0,0 +1,45 @@ +(: --------------------------------------------------------------------------- + aggregate-by-assettype.xq — XQuery 3.1 + + Portfolio exposure grouped by AssetType. Positions carry no AssetType; it + lives in AssetMasterData, so each Position is joined to its Asset by the + shared UniqueID (the standard FundsXML position<->asset link). + + FundsXML 4.x has no XML namespace — query bare element names. + + Output: with one per AssetType, sorted by + value descending, plus a grand total (sanity-check against 100% / fund NAV). +--------------------------------------------------------------------------- :) +declare option saxon:output "indent=yes"; + +let $doc := /FundsXML4 +let $fund := $doc/Funds/Fund[1] +let $ccy := $fund/Currency +let $assets := $doc/AssetMasterData/Asset +let $positions := $fund/FundDynamicData/Portfolios/Portfolio/Positions/Position + +return + +{ + for $p in $positions + let $a := $assets[UniqueID = $p/UniqueID] + let $type := ($a/AssetType, 'UNKNOWN')[1] + group by $type + let $value := sum($p/TotalValue/Amount[@ccy = $ccy] ! number(.)) + let $pct := sum($p/TotalPercentage ! number(.)) + order by $value descending + return + + {count($p)} + {format-number($value, '#0.00')} + {format-number($pct, '#0.0000')} + +} + + {count($positions)} + {format-number( + sum($positions/TotalValue/Amount[@ccy = $ccy] ! number(.)), '#0.00')} + {format-number( + sum($positions/TotalPercentage ! number(.)), '#0.0000')} + + diff --git a/XQuery_Examples/fund-summary.xq b/XQuery_Examples/fund-summary.xq new file mode 100644 index 0000000..af1999d --- /dev/null +++ b/XQuery_Examples/fund-summary.xq @@ -0,0 +1,54 @@ +(: --------------------------------------------------------------------------- + fund-summary.xq — XQuery 3.1 + + One-shot fund overview: identifiers, currency, NAV, share classes and a + reconciliation block (sum of position values vs. fund TotalNetAssetValue; + sum of position percentages). Useful as a quick integration health check. + + Works across versions (4.2.9 / 4.1.0 / 4.0.0) — backward compatible, and + ControlData/Version is simply absent for 4.0.0. + + FundsXML 4.x has no XML namespace — query bare element names. +--------------------------------------------------------------------------- :) +declare option saxon:output "indent=yes"; + +let $doc := /FundsXML4 +let $cd := $doc/ControlData +let $fund := $doc/Funds/Fund[1] +let $ccy := $fund/Currency +let $nav := number($fund/FundDynamicData/TotalAssetValues/TotalAssetValue + /TotalNetAssetValue/Amount[@ccy = $ccy]) +let $posSum := sum($fund/FundDynamicData/Portfolios/Portfolio/Positions/Position + /TotalValue/Amount[@ccy = $ccy] ! number(.)) +let $pctSum := sum($fund/FundDynamicData/Portfolios/Portfolio/Positions/Position + /TotalPercentage ! number(.)) + +return + + + {string($cd/UniqueDocumentID)} + {string(($cd/Version, 'n/a (4.0.0 has no Version element)')[1])} + {string($cd/ContentDate)} + + + {string($fund/Names/OfficialName)} + {string(($fund/Identifiers/LEI, '—')[1])} + {string($ccy)} + {format-number($nav, '#0.00')} + {count($fund/FundDynamicData/Portfolios/Portfolio/Positions/Position)} + + { + for $sc in $fund/SingleFund/ShareClasses/ShareClass + return + {string($sc/Names/OfficialName)} + } + + + + {format-number($posSum, '#0.00')} + {format-number(abs($posSum - $nav), '#0.00')} + {format-number($pctSum, '#0.0000')} + {abs($pctSum - 100) <= 1} + + diff --git a/XQuery_Examples/invocation/RunXQuery.java b/XQuery_Examples/invocation/RunXQuery.java new file mode 100644 index 0000000..63e549d --- /dev/null +++ b/XQuery_Examples/invocation/RunXQuery.java @@ -0,0 +1,48 @@ +// Run an XQuery against FundsXML via the Saxon s9api (native Java, no JAXB). +// +// tools/fetch-tools.sh +// SCP=.lib/Saxon-HE-12.5.jar:.lib/xmlresolver-5.2.2.jar:.lib/xmlresolver-5.2.2-data.jar +// javac -cp "$SCP" -d /tmp/xq XQuery_Examples/invocation/RunXQuery.java +// java -cp "$SCP:/tmp/xq" RunXQuery [k=v ...] +// Exit: 0 success, 2 setup error. +// +// External query variables are passed as k=v args (string-typed), e.g. n=5. + +import java.io.File; +import net.sf.saxon.s9api.Processor; +import net.sf.saxon.s9api.QName; +import net.sf.saxon.s9api.Serializer; +import net.sf.saxon.s9api.XQueryCompiler; +import net.sf.saxon.s9api.XQueryEvaluator; +import net.sf.saxon.s9api.XQueryExecutable; +import net.sf.saxon.s9api.XdmAtomicValue; + +public class RunXQuery { + public static void main(String[] args) { + if (args.length < 2) { + System.err.println("usage: RunXQuery [k=v ...]"); + System.exit(2); + } + try { + Processor proc = new Processor(false); + XQueryCompiler comp = proc.newXQueryCompiler(); + XQueryExecutable exe = comp.compile(new File(args[0])); + XQueryEvaluator qe = exe.load(); + + qe.setSource(new javax.xml.transform.stream.StreamSource(new File(args[1]))); + for (int i = 2; i < args.length; i++) { + int eq = args[i].indexOf('='); + if (eq > 0) { + qe.setExternalVariable(new QName(args[i].substring(0, eq)), + new XdmAtomicValue(args[i].substring(eq + 1))); + } + } + Serializer out = proc.newSerializer(System.out); + out.setOutputProperty(Serializer.Property.INDENT, "yes"); + qe.run(out); + } catch (Exception e) { + System.err.println("error: " + e.getMessage()); + System.exit(2); + } + } +} diff --git a/XQuery_Examples/invocation/run-xquery.sh b/XQuery_Examples/invocation/run-xquery.sh new file mode 100755 index 0000000..f6be744 --- /dev/null +++ b/XQuery_Examples/invocation/run-xquery.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Run an XQuery against a FundsXML document via Saxon-HE (CLI). +# +# Usage: run-xquery.sh [output] [name=value ...] +# Exit: 0 on success, 2 setup error, Saxon's code otherwise. +# +# Saxon's net.sf.saxon.Query takes external variables as bare `name=value` +# arguments (NOT `-param:` — that is the XSLT entry point; and a leading `!` +# would set serialization params instead). tools/fetch-tools.sh provides +# Saxon-HE in .lib/. +set -euo pipefail + +Q="${1:?usage: run-xquery.sh [output] [name=value ...]}" +IN="${2:?input xml required}" +OUT="${3:-}" +shift 2 || true +[[ $# -ge 1 && "${1:-}" != *=* ]] && { OUT="$1"; shift; } + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +LIB="${REPO_ROOT}/.lib" +if [[ ! -f "${LIB}/Saxon-HE-12.5.jar" ]]; then + echo "Saxon not present; fetching..." >&2 + "${REPO_ROOT}/tools/fetch-tools.sh" >/dev/null +fi +SCP="${LIB}/Saxon-HE-12.5.jar:${LIB}/xmlresolver-5.2.2.jar:${LIB}/xmlresolver-5.2.2-data.jar" + +if [[ -n "$OUT" ]]; then + java -cp "$SCP" net.sf.saxon.Query -q:"$Q" -s:"$IN" -o:"$OUT" "$@" + echo "wrote $OUT" +else + java -cp "$SCP" net.sf.saxon.Query -q:"$Q" -s:"$IN" "$@" +fi diff --git a/XQuery_Examples/invocation/run_xquery.py b/XQuery_Examples/invocation/run_xquery.py new file mode 100644 index 0000000..6ac313c --- /dev/null +++ b/XQuery_Examples/invocation/run_xquery.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Run an XQuery against FundsXML via saxonche (Saxon's Python API). + +Usage: python run_xquery.py [k=v ...] +Exit: 0 success, 2 setup error. + +`pip install saxonche` provides Saxon's XQuery 3.1 engine. lxml has no XQuery. +External query variables are passed as k=v args (string-typed), e.g. n=5. +""" +import sys + + +def main() -> int: + if len(sys.argv) < 3: + print("usage: run_xquery.py [k=v ...]", + file=sys.stderr) + return 2 + query, src = sys.argv[1], sys.argv[2] + params = dict(p.split("=", 1) for p in sys.argv[3:] if "=" in p) + + try: + from saxonche import PySaxonProcessor + except ImportError: + print("saxonche not installed. Run: pip install saxonche", + file=sys.stderr) + return 2 + + with PySaxonProcessor(license=False) as proc: + xq = proc.new_xquery_processor() + xq.set_context(file_name=src) + for k, v in params.items(): + xq.set_parameter(k, proc.make_string_value(v)) + xq.set_query_file(query) + sys.stdout.write(xq.run_query_to_string()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/XQuery_Examples/look-through.xq b/XQuery_Examples/look-through.xq new file mode 100644 index 0000000..9771367 --- /dev/null +++ b/XQuery_Examples/look-through.xq @@ -0,0 +1,47 @@ +(: --------------------------------------------------------------------------- + look-through.xq — XQuery 3.1 + + Fund-of-funds look-through readiness. Positions whose Asset is of type SC + (a fund / share-class investment) are not "final" holdings — to compute true + exposure you would substitute the target fund's own portfolio, weighted by + this position's weight. A single FundsXML file rarely contains the nested + funds, so this query reports the look-through *candidates* and the residual + directly-held portion. + + FundsXML 4.x has no XML namespace — query bare element names. +--------------------------------------------------------------------------- :) +declare option saxon:output "indent=yes"; + +let $doc := /FundsXML4 +let $fund := $doc/Funds/Fund[1] +let $ccy := $fund/Currency +let $assets := $doc/AssetMasterData/Asset +let $pos := $fund/FundDynamicData/Portfolios/Portfolio/Positions/Position + +let $scPos := $pos[ some $a in $assets + satisfies $a/UniqueID = ./UniqueID and $a/AssetType = 'SC' ] +let $scPct := sum($scPos/TotalPercentage ! number(.)) + +return + + + {count($scPos)} + {format-number($scPct div 100, '0.00%')} + {format-number((100 - $scPct) div 100, '0.00%')} + + + { + for $p in $scPos + let $a := $assets[UniqueID = $p/UniqueID] + order by number($p/TotalPercentage) descending + return + + {string(($p/Identifiers/ISIN, $a/Identifiers/ISIN, '—')[1])} + {string(($a/Name, '—')[1])} + {string(($a/AssetDetails/ShareClass/Issuer/Name, '—')[1])} + {format-number(number($p/TotalPercentage) div 100, '0.00%')} + substitute target fund portfolio, scaled by this weight + + } + + diff --git a/XQuery_Examples/top-holdings.xq b/XQuery_Examples/top-holdings.xq new file mode 100644 index 0000000..5d4abd5 --- /dev/null +++ b/XQuery_Examples/top-holdings.xq @@ -0,0 +1,37 @@ +(: --------------------------------------------------------------------------- + top-holdings.xq — XQuery 3.1 + + The N largest holdings by value in fund currency, with the asset name + resolved from AssetMasterData (Position<->Asset joined by UniqueID). + + External parameter: $n (number of holdings, default 10) + Saxon CLI: -param:n=5 + s9api: qe.setExternalVariable(new QName("n"), new XdmAtomicValue(5)) + + FundsXML 4.x has no XML namespace — query bare element names. +--------------------------------------------------------------------------- :) +declare option saxon:output "indent=yes"; +declare variable $n external := 10; + +let $doc := /FundsXML4 +let $fund := $doc/Funds/Fund[1] +let $ccy := $fund/Currency +let $assets := $doc/AssetMasterData/Asset + +return + +{ + for $p in $fund/FundDynamicData/Portfolios/Portfolio/Positions/Position + let $v := number($p/TotalValue/Amount[@ccy = $ccy]) + order by $v descending + count $rank + where $rank <= xs:integer($n) + return + + {string(($p/Identifiers/ISIN, $p/UniqueID)[1])} + {string(($assets[UniqueID = $p/UniqueID]/Name, '—')[1])} + {format-number($v, '#0.00')} + {format-number(number($p/TotalPercentage) div 100, '0.00%')} + +} +