Phase 2: XQuery analytics (aggregation, top-holdings, look-through)

.github/workflows/ci.yml

@@ -86,6 +86,20 @@ jobs:
           done
           test "$(wc -l < out_pos.csv)" -ge 2
 
+      - name: Smoke - XQuery examples run and produce XML
+        run: |
+          set -e
+          RQ=XQuery_Examples/invocation/run-xquery.sh
+          SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml
+          "$RQ" XQuery_Examples/aggregate-by-assettype.xq "$SRC" out_agg.xml
+          "$RQ" XQuery_Examples/top-holdings.xq "$SRC" out_top.xml n=5
+          "$RQ" XQuery_Examples/fund-summary.xq "$SRC" out_sum.xml
+          "$RQ" XQuery_Examples/look-through.xq "$SRC" out_lt.xml
+          for f in out_agg.xml out_top.xml out_sum.xml out_lt.xml; do
+            xmllint --noout "$f"
+          done
+          test "$(xmllint --xpath 'count(//Holding)' out_top.xml)" = "5"
+
       - name: Regression - legacy XSLT 1.0 report still runs
         run: |
           xsltproc XSLT_DataQuality_Checks/Enhanced_Check/FundsXML_CompleteDQReport_HTML.xsl \

.gitignore

@@ -17,4 +17,5 @@ out/
 out_*.html
 out_*.fo
 out_*.csv
+out_*.xml
 *.svrl

README.md

@@ -35,7 +35,7 @@ locations. Items marked _(planned)_ are on the roadmap (see
 | Transformation invocation | CLI, Python, Java, .NET, Node | [XSLT_Transformations/invocation/](./XSLT_Transformations/) | ✅ |
 | Schema fetch (proxy-aware) | Bash | [tools/fetch-schema.sh](./tools/fetch-schema.sh) | ✅ |
 | CI (validate all samples) | GitHub Actions | [.github/workflows/ci.yml](./.github/workflows/) | ✅ |
-| XQuery examples | BaseX/Saxon, Python, Java, .NET | `XQuery_Examples/` | _(planned)_ |
+| XQuery analytics (aggregation, top-holdings, look-through) | Saxon CLI/Java, Python, BaseX | [XQuery_Examples/](./XQuery_Examples/) | ✅ |
 | XML signature sign/verify | Apache Santuario, .NET, xmlsec1, signxml | `XML_Signature/` | _(planned)_ |
 | Database load ↔ generate | Oracle/SQL Server/Postgres (code only) | `Database_Integration/` | _(planned)_ |
 | Large-file/stream processing | StAX/SAX/lxml iterparse | `Large_File_Processing/` | _(planned)_ |

XQuery_Examples/README.md

@@ -0,0 +1,47 @@
+# XQuery Examples
+
+![XQuery](https://img.shields.io/badge/XQuery-3.1-blue) ![status](https://img.shields.io/badge/status-verified-brightgreen)
+
+Read-only analytics over FundsXML with XQuery 3.1. Positions and assets are
+joined by the shared `UniqueID` (the standard FundsXML link). FundsXML 4.x has
+no XML namespace — queries use bare element names, so they work unchanged on the
+4.2.9 / 4.1.0 / 4.0.0 samples.
+
+| Query | Purpose | Notable |
+|-------|---------|---------|
+| [`aggregate-by-assettype.xq`](aggregate-by-assettype.xq) | Exposure grouped by `AssetType` | `group by`, grand-total reconciliation |
+| [`top-holdings.xq`](top-holdings.xq) | N largest holdings, names resolved | external var `n` (default 10) |
+| [`fund-summary.xq`](fund-summary.xq) | Fund/doc overview + reconciliation block | version-agnostic (4.0.0 has no `Version`) |
+| [`look-through.xq`](look-through.xq) | Fund-of-funds look-through readiness | flags `SC` (fund) positions + weights |
+
+## Run (per stack)
+
+| Stack | Entry point | Status |
+|-------|-------------|--------|
+| CLI (Saxon) | [`invocation/run-xquery.sh`](invocation/run-xquery.sh) | ✅ verified |
+| Java (s9api, no JAXB) | [`invocation/RunXQuery.java`](invocation/RunXQuery.java) | ✅ verified |
+| Python | [`invocation/run_xquery.py`](invocation/run_xquery.py) | needs `pip install saxonche` |
+| BaseX | see below | needs BaseX install |
+
+```bash
+tools/fetch-tools.sh
+SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml
+
+# CLI — to stdout
+XQuery_Examples/invocation/run-xquery.sh XQuery_Examples/fund-summary.xq "$SRC"
+
+# CLI — external variable + output file (Saxon Query: bare name=value)
+XQuery_Examples/invocation/run-xquery.sh XQuery_Examples/top-holdings.xq "$SRC" top.xml n=5
+
+# Java s9api
+SCP=.lib/Saxon-HE-12.5.jar:.lib/xmlresolver-5.2.2.jar:.lib/xmlresolver-5.2.2-data.jar
+javac -cp "$SCP" -d /tmp/xq XQuery_Examples/invocation/RunXQuery.java
+java  -cp "$SCP:/tmp/xq" RunXQuery XQuery_Examples/aggregate-by-assettype.xq "$SRC"
+
+# BaseX (in-memory, binds the document and the external variable)
+basex -i "$SRC" -bn=5 XQuery_Examples/top-holdings.xq
+```
+
+Note: Saxon's `net.sf.saxon.Query` takes external variables as plain
+`name=value` arguments — **not** `-param:` (that is the XSLT entry point) and
+not `!name=value` (that sets serialization properties).

XQuery_Examples/aggregate-by-assettype.xq

@@ -0,0 +1,45 @@
+(: ---------------------------------------------------------------------------
+   aggregate-by-assettype.xq  —  XQuery 3.1
+
+   Portfolio exposure grouped by AssetType. Positions carry no AssetType; it
+   lives in AssetMasterData, so each Position is joined to its Asset by the
+   shared UniqueID (the standard FundsXML position<->asset link).
+
+   FundsXML 4.x has no XML namespace — query bare element names.
+
+   Output: <ExposureByAssetType> with one <Bucket> per AssetType, sorted by
+   value descending, plus a grand total (sanity-check against 100% / fund NAV).
+--------------------------------------------------------------------------- :)
+declare option saxon:output "indent=yes";
+
+let $doc      := /FundsXML4
+let $fund     := $doc/Funds/Fund[1]
+let $ccy      := $fund/Currency
+let $assets   := $doc/AssetMasterData/Asset
+let $positions := $fund/FundDynamicData/Portfolios/Portfolio/Positions/Position
+
+return
+<ExposureByAssetType fund="{$fund/Names/OfficialName}" currency="{$ccy}">
+{
+  for $p in $positions
+  let $a    := $assets[UniqueID = $p/UniqueID]
+  let $type := ($a/AssetType, 'UNKNOWN')[1]
+  group by $type
+  let $value := sum($p/TotalValue/Amount[@ccy = $ccy] ! number(.))
+  let $pct   := sum($p/TotalPercentage ! number(.))
+  order by $value descending
+  return
+    <Bucket assetType="{$type}">
+      <Positions>{count($p)}</Positions>
+      <Value currency="{$ccy}">{format-number($value, '#0.00')}</Value>
+      <Percentage>{format-number($pct, '#0.0000')}</Percentage>
+    </Bucket>
+}
+  <Total>
+    <Positions>{count($positions)}</Positions>
+    <Value currency="{$ccy}">{format-number(
+      sum($positions/TotalValue/Amount[@ccy = $ccy] ! number(.)), '#0.00')}</Value>
+    <Percentage>{format-number(
+      sum($positions/TotalPercentage ! number(.)), '#0.0000')}</Percentage>
+  </Total>
+</ExposureByAssetType>

XQuery_Examples/fund-summary.xq

@@ -0,0 +1,54 @@
+(: ---------------------------------------------------------------------------
+   fund-summary.xq  —  XQuery 3.1
+
+   One-shot fund overview: identifiers, currency, NAV, share classes and a
+   reconciliation block (sum of position values vs. fund TotalNetAssetValue;
+   sum of position percentages). Useful as a quick integration health check.
+
+   Works across versions (4.2.9 / 4.1.0 / 4.0.0) — backward compatible, and
+   ControlData/Version is simply absent for 4.0.0.
+
+   FundsXML 4.x has no XML namespace — query bare element names.
+--------------------------------------------------------------------------- :)
+declare option saxon:output "indent=yes";
+
+let $doc   := /FundsXML4
+let $cd    := $doc/ControlData
+let $fund  := $doc/Funds/Fund[1]
+let $ccy   := $fund/Currency
+let $nav   := number($fund/FundDynamicData/TotalAssetValues/TotalAssetValue
+                      /TotalNetAssetValue/Amount[@ccy = $ccy])
+let $posSum := sum($fund/FundDynamicData/Portfolios/Portfolio/Positions/Position
+                   /TotalValue/Amount[@ccy = $ccy] ! number(.))
+let $pctSum := sum($fund/FundDynamicData/Portfolios/Portfolio/Positions/Position
+                   /TotalPercentage ! number(.))
+
+return
+<FundSummary>
+  <Document>
+    <Id>{string($cd/UniqueDocumentID)}</Id>
+    <Version>{string(($cd/Version, 'n/a (4.0.0 has no Version element)')[1])}</Version>
+    <ContentDate>{string($cd/ContentDate)}</ContentDate>
+  </Document>
+  <Fund>
+    <Name>{string($fund/Names/OfficialName)}</Name>
+    <LEI>{string(($fund/Identifiers/LEI, '—')[1])}</LEI>
+    <Currency>{string($ccy)}</Currency>
+    <TotalNetAssetValue>{format-number($nav, '#0.00')}</TotalNetAssetValue>
+    <Positions>{count($fund/FundDynamicData/Portfolios/Portfolio/Positions/Position)}</Positions>
+    <ShareClasses>
+    {
+      for $sc in $fund/SingleFund/ShareClasses/ShareClass
+      return <ShareClass isin="{$sc/Identifiers/ISIN}" ccy="{$sc/Currency}"
+                         navPrice="{$sc/Prices/Price/NavPrice}">
+               {string($sc/Names/OfficialName)}</ShareClass>
+    }
+    </ShareClasses>
+  </Fund>
+  <Reconciliation>
+    <SumPositionValues>{format-number($posSum, '#0.00')}</SumPositionValues>
+    <NavDelta>{format-number(abs($posSum - $nav), '#0.00')}</NavDelta>
+    <SumPercentages>{format-number($pctSum, '#0.0000')}</SumPercentages>
+    <PercentageOk>{abs($pctSum - 100) <= 1}</PercentageOk>
+  </Reconciliation>
+</FundSummary>

XQuery_Examples/invocation/RunXQuery.java

@@ -0,0 +1,48 @@
+// Run an XQuery against FundsXML via the Saxon s9api (native Java, no JAXB).
+//
+//   tools/fetch-tools.sh
+//   SCP=.lib/Saxon-HE-12.5.jar:.lib/xmlresolver-5.2.2.jar:.lib/xmlresolver-5.2.2-data.jar
+//   javac -cp "$SCP" -d /tmp/xq XQuery_Examples/invocation/RunXQuery.java
+//   java  -cp "$SCP:/tmp/xq" RunXQuery <query.xq> <input.xml> [k=v ...]
+// Exit: 0 success, 2 setup error.
+//
+// External query variables are passed as k=v args (string-typed), e.g. n=5.
+
+import java.io.File;
+import net.sf.saxon.s9api.Processor;
+import net.sf.saxon.s9api.QName;
+import net.sf.saxon.s9api.Serializer;
+import net.sf.saxon.s9api.XQueryCompiler;
+import net.sf.saxon.s9api.XQueryEvaluator;
+import net.sf.saxon.s9api.XQueryExecutable;
+import net.sf.saxon.s9api.XdmAtomicValue;
+
+public class RunXQuery {
+    public static void main(String[] args) {
+        if (args.length < 2) {
+            System.err.println("usage: RunXQuery <query.xq> <input.xml> [k=v ...]");
+            System.exit(2);
+        }
+        try {
+            Processor proc = new Processor(false);
+            XQueryCompiler comp = proc.newXQueryCompiler();
+            XQueryExecutable exe = comp.compile(new File(args[0]));
+            XQueryEvaluator qe = exe.load();
+
+            qe.setSource(new javax.xml.transform.stream.StreamSource(new File(args[1])));
+            for (int i = 2; i < args.length; i++) {
+                int eq = args[i].indexOf('=');
+                if (eq > 0) {
+                    qe.setExternalVariable(new QName(args[i].substring(0, eq)),
+                                           new XdmAtomicValue(args[i].substring(eq + 1)));
+                }
+            }
+            Serializer out = proc.newSerializer(System.out);
+            out.setOutputProperty(Serializer.Property.INDENT, "yes");
+            qe.run(out);
+        } catch (Exception e) {
+            System.err.println("error: " + e.getMessage());
+            System.exit(2);
+        }
+    }
+}

XQuery_Examples/invocation/run-xquery.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Run an XQuery against a FundsXML document via Saxon-HE (CLI).
+#
+# Usage: run-xquery.sh <query.xq> <input.xml> [output] [name=value ...]
+# Exit:  0 on success, 2 setup error, Saxon's code otherwise.
+#
+# Saxon's net.sf.saxon.Query takes external variables as bare `name=value`
+# arguments (NOT `-param:` — that is the XSLT entry point; and a leading `!`
+# would set serialization params instead). tools/fetch-tools.sh provides
+# Saxon-HE in .lib/.
+set -euo pipefail
+
+Q="${1:?usage: run-xquery.sh <query.xq> <input.xml> [output] [name=value ...]}"
+IN="${2:?input xml required}"
+OUT="${3:-}"
+shift 2 || true
+[[ $# -ge 1 && "${1:-}" != *=* ]] && { OUT="$1"; shift; }
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+LIB="${REPO_ROOT}/.lib"
+if [[ ! -f "${LIB}/Saxon-HE-12.5.jar" ]]; then
+  echo "Saxon not present; fetching..." >&2
+  "${REPO_ROOT}/tools/fetch-tools.sh" >/dev/null
+fi
+SCP="${LIB}/Saxon-HE-12.5.jar:${LIB}/xmlresolver-5.2.2.jar:${LIB}/xmlresolver-5.2.2-data.jar"
+
+if [[ -n "$OUT" ]]; then
+  java -cp "$SCP" net.sf.saxon.Query -q:"$Q" -s:"$IN" -o:"$OUT" "$@"
+  echo "wrote $OUT"
+else
+  java -cp "$SCP" net.sf.saxon.Query -q:"$Q" -s:"$IN" "$@"
+fi

XQuery_Examples/invocation/run_xquery.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+"""Run an XQuery against FundsXML via saxonche (Saxon's Python API).
+
+Usage:  python run_xquery.py <query.xq> <input.xml> [k=v ...]
+Exit:   0 success, 2 setup error.
+
+`pip install saxonche` provides Saxon's XQuery 3.1 engine. lxml has no XQuery.
+External query variables are passed as k=v args (string-typed), e.g. n=5.
+"""
+import sys
+
+
+def main() -> int:
+    if len(sys.argv) < 3:
+        print("usage: run_xquery.py <query.xq> <input.xml> [k=v ...]",
+              file=sys.stderr)
+        return 2
+    query, src = sys.argv[1], sys.argv[2]
+    params = dict(p.split("=", 1) for p in sys.argv[3:] if "=" in p)
+
+    try:
+        from saxonche import PySaxonProcessor
+    except ImportError:
+        print("saxonche not installed. Run: pip install saxonche",
+              file=sys.stderr)
+        return 2
+
+    with PySaxonProcessor(license=False) as proc:
+        xq = proc.new_xquery_processor()
+        xq.set_context(file_name=src)
+        for k, v in params.items():
+            xq.set_parameter(k, proc.make_string_value(v))
+        xq.set_query_file(query)
+        sys.stdout.write(xq.run_query_to_string())
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

XQuery_Examples/look-through.xq

@@ -0,0 +1,47 @@
+(: ---------------------------------------------------------------------------
+   look-through.xq  —  XQuery 3.1
+
+   Fund-of-funds look-through readiness. Positions whose Asset is of type SC
+   (a fund / share-class investment) are not "final" holdings — to compute true
+   exposure you would substitute the target fund's own portfolio, weighted by
+   this position's weight. A single FundsXML file rarely contains the nested
+   funds, so this query reports the look-through *candidates* and the residual
+   directly-held portion.
+
+   FundsXML 4.x has no XML namespace — query bare element names.
+--------------------------------------------------------------------------- :)
+declare option saxon:output "indent=yes";
+
+let $doc    := /FundsXML4
+let $fund   := $doc/Funds/Fund[1]
+let $ccy    := $fund/Currency
+let $assets := $doc/AssetMasterData/Asset
+let $pos    := $fund/FundDynamicData/Portfolios/Portfolio/Positions/Position
+
+let $scPos  := $pos[ some $a in $assets
+                     satisfies $a/UniqueID = ./UniqueID and $a/AssetType = 'SC' ]
+let $scPct  := sum($scPos/TotalPercentage ! number(.))
+
+return
+<LookThrough fund="{$fund/Names/OfficialName}" currency="{$ccy}">
+  <Summary>
+    <FundInvestments>{count($scPos)}</FundInvestments>
+    <LookThroughWeight>{format-number($scPct div 100, '0.00%')}</LookThroughWeight>
+    <DirectlyHeldWeight>{format-number((100 - $scPct) div 100, '0.00%')}</DirectlyHeldWeight>
+  </Summary>
+  <Candidates>
+  {
+    for $p in $scPos
+    let $a := $assets[UniqueID = $p/UniqueID]
+    order by number($p/TotalPercentage) descending
+    return
+      <Candidate>
+        <Isin>{string(($p/Identifiers/ISIN, $a/Identifiers/ISIN, '—')[1])}</Isin>
+        <Name>{string(($a/Name, '—')[1])}</Name>
+        <Issuer>{string(($a/AssetDetails/ShareClass/Issuer/Name, '—')[1])}</Issuer>
+        <Weight>{format-number(number($p/TotalPercentage) div 100, '0.00%')}</Weight>
+        <Note>substitute target fund portfolio, scaled by this weight</Note>
+      </Candidate>
+  }
+  </Candidates>
+</LookThrough>

XQuery_Examples/top-holdings.xq

@@ -0,0 +1,37 @@
+(: ---------------------------------------------------------------------------
+   top-holdings.xq  —  XQuery 3.1
+
+   The N largest holdings by value in fund currency, with the asset name
+   resolved from AssetMasterData (Position<->Asset joined by UniqueID).
+
+   External parameter:  $n  (number of holdings, default 10)
+     Saxon CLI:  -param:n=5
+     s9api:      qe.setExternalVariable(new QName("n"), new XdmAtomicValue(5))
+
+   FundsXML 4.x has no XML namespace — query bare element names.
+--------------------------------------------------------------------------- :)
+declare option saxon:output "indent=yes";
+declare variable $n external := 10;
+
+let $doc    := /FundsXML4
+let $fund   := $doc/Funds/Fund[1]
+let $ccy    := $fund/Currency
+let $assets := $doc/AssetMasterData/Asset
+
+return
+<TopHoldings fund="{$fund/Names/OfficialName}" currency="{$ccy}" n="{$n}">
+{
+  for $p in $fund/FundDynamicData/Portfolios/Portfolio/Positions/Position
+  let $v := number($p/TotalValue/Amount[@ccy = $ccy])
+  order by $v descending
+  count $rank
+  where $rank <= xs:integer($n)
+  return
+    <Holding rank="{$rank}">
+      <Id>{string(($p/Identifiers/ISIN, $p/UniqueID)[1])}</Id>
+      <Name>{string(($assets[UniqueID = $p/UniqueID]/Name, '—')[1])}</Name>
+      <Value currency="{$ccy}">{format-number($v, '#0.00')}</Value>
+      <Weight>{format-number(number($p/TotalPercentage) div 100, '0.00%')}</Weight>
+    </Holding>
+}
+</TopHoldings>