Skip to content

fix: small code defects from the full example review + documentation sweep #82

fix: small code defects from the full example review + documentation sweep

fix: small code defects from the full example review + documentation sweep #82

Workflow file for this run

name: FundsXML CI
# Validates every committed sample against the official released XSD and the
# Schematron ruleset, runs the XSLT transforms as a smoke test, and asserts the
# negative fixtures still fail. Positive samples must stay green.
#
# Security note: no untrusted GitHub event data (issue/PR/commit text) is used
# in any run: step, so there is no workflow-injection surface here.
on:
push:
branches: ["**"]
pull_request:
workflow_dispatch:
jobs:
validate:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Install tooling
run: sudo apt-get update -qq && sudo apt-get install -y -qq libxml2-utils xsltproc
- uses: actions/setup-java@v5
with:
distribution: temurin
java-version: "21"
- uses: actions/setup-python@v6
with:
python-version: "3.11"
- uses: actions/setup-node@v6
with:
node-version: "20"
- uses: actions/setup-dotnet@v5
with:
dotnet-version: "8.0"
# One idiomatic, OS-agnostic Python install (venv + editable pyproject).
# Dependency-only manifest — nothing is packaged/importable.
- name: Python - venv & install (pyproject)
run: |
python -m venv .venv
.venv/bin/pip install --quiet --upgrade pip
.venv/bin/pip install --quiet -e .
.venv/bin/python -c "import lxml, saxonche; print('python deps OK')"
# The examples no longer resolve any schema: each validator takes a
# schema path (or URL) + the XML file. This step is CI plumbing only —
# a plain curl of the official releases into ci-schemas/ so the many
# raw-xmllint steps below have a local schema path to point at. (The
# validators' own remote-URL handling is exercised separately.)
- name: Fetch official schemas for CI (plain curl, no resolver)
run: |
set -e
for v in 4.2.9 4.1.0 4.0.0; do
mkdir -p "ci-schemas/$v"
curl -sSL --fail -o "ci-schemas/$v/FundsXML.xsd" \
"https://github.com/fundsxml/schema/releases/download/$v/FundsXML.xsd"
if grep -q 'xmldsig-core-schema\.xsd' "ci-schemas/$v/FundsXML.xsd"; then
curl -sSL --fail -o "ci-schemas/$v/xmldsig-core-schema.xsd" \
"https://github.com/fundsxml/schema/releases/download/$v/xmldsig-core-schema.xsd"
fi
done
- name: XSD - all positive samples must validate
run: |
set -e
declare -A S=(
["4.2.9"]="positions/Mixed-Fund_Positions transactions/Fund_Transactions documents/Fund_Documents regulatory/EFT_Regulatory signed/Signed_Fund_Skeleton"
["4.1.0"]="positions/Equity-Fund_Positions"
["4.0.0"]="positions/Equity-Fund_Positions"
)
for v in "${!S[@]}"; do
for f in ${S[$v]}; do
echo "XSD $v/$f"
xmllint --noout --nonet --schema "ci-schemas/$v/FundsXML.xsd" \
"FundsXML_Files/$v/$f.xml"
done
done
- name: XSD - negative fixture must FAIL
run: |
if xmllint --noout --nonet --schema ci-schemas/4.2.9/FundsXML.xsd \
tests/fixtures/invalid/xsd-invalid_Positions.xml 2>/dev/null; then
echo "::error::xsd-invalid fixture unexpectedly validated"; exit 1
fi
echo "xsd-invalid correctly rejected"
- name: Python - XSD validation (schema path + remote URL)
run: |
set -e
V=".venv/bin/python XSD_Validation/python/validate.py"
# Local schema path (two versions) ...
$V ci-schemas/4.2.9/FundsXML.xsd FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml
$V ci-schemas/4.1.0/FundsXML.xsd FundsXML_Files/4.1.0/positions/Equity-Fund_Positions.xml
# ... and a remote schema URL (the official release, fetched by the
# validator itself — exercises the URL path + the relative xmldsig
# sibling fetch).
$V https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd \
FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml
if $V ci-schemas/4.2.9/FundsXML.xsd tests/fixtures/invalid/xsd-invalid_Positions.xml; then
echo "::error::xsd-invalid unexpectedly validated (Python)"; exit 1
fi
echo "Python validate.py: path + URL ok, negative correctly rejected"
- name: .NET - XSD validation (schema path + remote URL)
run: |
set -e
D="dotnet run --project XSD_Validation/dotnet --"
$D ci-schemas/4.2.9/FundsXML.xsd FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml
$D ci-schemas/4.1.0/FundsXML.xsd FundsXML_Files/4.1.0/positions/Equity-Fund_Positions.xml
# Remote schema URL: .NET resolves it (and the relative xmldsig
# import) natively via the schema-set URL resolver.
$D https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd \
FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml
if $D ci-schemas/4.2.9/FundsXML.xsd tests/fixtures/invalid/xsd-invalid_Positions.xml; then
echo "::error::xsd-invalid unexpectedly validated (.NET)"; exit 1
fi
echo ".NET XsdValidate: path + URL ok, negative correctly rejected"
- name: CLI - validate.sh (schema path + remote URL)
run: |
set -e
XSD_Validation/cli/validate.sh ci-schemas/4.2.9/FundsXML.xsd \
FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml
# Remote schema URL: validate.sh fetches it (and the xmldsig
# sibling) into a temp dir, then validates offline with --nonet.
XSD_Validation/cli/validate.sh \
https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd \
FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml
if XSD_Validation/cli/validate.sh ci-schemas/4.2.9/FundsXML.xsd \
tests/fixtures/invalid/xsd-invalid_Positions.xml; then
echo "::error::xsd-invalid unexpectedly validated (CLI)"; exit 1
fi
echo "CLI validate.sh: path + URL ok, negative correctly rejected"
# ---------------------------------------------------------------------
# Java examples — built & run via the committed Maven Wrapper. The first
# ./mvnw bootstraps Maven itself, then resolves all deps from Central.
# A repo-wide `compile` builds every module once so later `exec:java`
# steps need no `compile` goal (keeps stdout clean for the XQuery step,
# which serializes its result to stdout).
# ---------------------------------------------------------------------
- name: Java - build all modules (Maven Wrapper)
run: ./mvnw -q -B compile
- name: Java - XSD validation (schema path + remote URL)
run: |
set -e
M="./mvnw -q -B -pl XSD_Validation/java exec:java"
$M -Dexec.args="ci-schemas/4.2.9/FundsXML.xsd FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml"
$M -Dexec.args="ci-schemas/4.1.0/FundsXML.xsd FundsXML_Files/4.1.0/positions/Equity-Fund_Positions.xml"
# Remote schema URL: the validator fetches it (and the xmldsig
# sibling) into a temp dir, then validates locally.
$M -Dexec.args="https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml"
if $M -Dexec.args="ci-schemas/4.2.9/FundsXML.xsd tests/fixtures/invalid/xsd-invalid_Positions.xml"; then
echo "::error::xsd-invalid unexpectedly validated (Java)"; exit 1
fi
echo "Java XsdValidate: path + URL ok, negative correctly rejected"
- name: Schematron - canonical sample passes, fixture fails
run: |
set -e
SCH=Schematron_DataQuality_Checks/Basic_Checks/basic_checks.sch
M="./mvnw -q -B -pl Schematron_DataQuality_Checks/Basic_Checks/invocation exec:java"
$M -Dexec.args="$SCH FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml"
if $M -Dexec.args="$SCH tests/fixtures/invalid/schematron-invalid_Positions.xml"; then
echo "::error::schematron-invalid fixture unexpectedly passed"; exit 1
fi
echo "schematron-invalid correctly failed"
- name: Smoke - XSLT transforms produce non-empty output
run: |
set -e
M="./mvnw -q -B -pl XSLT_Transformations/invocation exec:java"
SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml
$M -Dexec.args="XSLT_DataQuality_Checks/Custom_Internal_Checks/custom_internal_checks.xslt $SRC out_dq.html"
$M -Dexec.args="XSLT_Transformations/Factsheet/factsheet_html.xslt $SRC out_fs.html"
$M -Dexec.args="XSLT_Transformations/Factsheet/factsheet_fo.xslt $SRC out_fs.fo"
$M -Dexec.args="XSLT_Transformations/CSV_Export/positions_csv.xslt $SRC out_pos.csv"
xmllint --noout out_fs.fo
for f in out_dq.html out_fs.html out_pos.csv; do
test -s "$f" || { echo "::error::$f is empty"; exit 1; }
done
test "$(wc -l < out_pos.csv)" -ge 2
- name: Smoke - XQuery examples run and produce XML
run: |
set -e
M="./mvnw -q -B -pl XQuery_Examples/invocation exec:java"
SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml
# RunXQuery serializes to stdout; -q keeps Maven silent so the
# redirect captures pure XML.
$M -Dexec.args="XQuery_Examples/aggregate-by-assettype.xq $SRC" > out_agg.xml
$M -Dexec.args="XQuery_Examples/top-holdings.xq $SRC n=5" > out_top.xml
$M -Dexec.args="XQuery_Examples/fund-summary.xq $SRC" > out_sum.xml
$M -Dexec.args="XQuery_Examples/look-through.xq $SRC" > out_lt.xml
for f in out_agg.xml out_top.xml out_sum.xml out_lt.xml; do
xmllint --noout "$f"
done
test "$(xmllint --xpath 'count(//Holding)' out_top.xml)" = "5"
- name: XML signature - Apache Santuario sign/verify/tamper roundtrip
run: |
set -e
M="./mvnw -q -B -pl XML_Signature/java exec:java"
SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml
# In-language key generation (JDK keytool, no openssl).
$M -Dexec.mainClass=GenerateTestKey
$M -Dexec.mainClass=SignFundsXml \
-Dexec.args="$SRC signed.xml XML_Signature/keys/test-signing.p12 changeit fundsxml"
$M -Dexec.mainClass=VerifyFundsXml \
-Dexec.args="signed.xml XML_Signature/keys/test-signing-cert.pem"
xmllint --noout --nonet --schema ci-schemas/4.2.9/FundsXML.xsd signed.xml
sed 's/<TotalPercentage>8.33</<TotalPercentage>9.33</' signed.xml > tampered.xml
if $M -Dexec.mainClass=VerifyFundsXml \
-Dexec.args="tampered.xml XML_Signature/keys/test-signing-cert.pem"; then
echo "::error::tampered file unexpectedly verified"; exit 1
fi
echo "tamper correctly detected"
- name: DB integration - multi-fund import+export in 4 languages
run: |
set -e
PY=.venv/bin/python
FX=FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml
MX=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml
DOC=FUNDSXML_MULTI_1
EQ="$PY Database_Integration/tools/xml_equiv.py"
XSD="ci-schemas/4.2.9/FundsXML.xsd"
# The multi-fund fixture must itself be schema-valid.
xmllint --noout --nonet --schema "$XSD" "$FX"
# Each language: separate import then export, then prove
# exported file == input file (and is schema-valid).
# Python
$PY Database_Integration/python/import_fundsxml.py py.db "$FX"
$PY Database_Integration/python/export_fundsxml.py py.db "$DOC" py.xml
xmllint --noout --nonet --schema "$XSD" py.xml
$EQ "$FX" py.xml
# Java (sqlite-jdbc via Maven; classes already built above)
MJ="./mvnw -q -B -pl Database_Integration/java exec:java"
$MJ -Dexec.mainClass=ImportFundsXml -Dexec.args="java.db $FX"
$MJ -Dexec.mainClass=ExportFundsXml -Dexec.args="java.db $DOC java.xml"
xmllint --noout --nonet --schema "$XSD" java.xml
$EQ "$FX" java.xml
# JavaScript (sql.js)
( cd Database_Integration/javascript && npm install --no-audit --no-fund )
node Database_Integration/javascript/import_fundsxml.mjs js.db "$FX"
node Database_Integration/javascript/export_fundsxml.mjs js.db "$DOC" js.xml
xmllint --noout --nonet --schema "$XSD" js.xml
$EQ "$FX" js.xml
# C# (.NET): separate import and export projects
dotnet run --project Database_Integration/csharp/import -- cs.db "$FX"
dotnet run --project Database_Integration/csharp/export -- cs.db "$DOC" cs.xml
xmllint --noout --nonet --schema "$XSD" cs.xml
$EQ "$FX" cs.xml
# Cross-language: every export identical to Python's.
$EQ py.xml java.xml && $EQ py.xml js.xml && $EQ py.xml cs.xml
# Single-fund sample still imports+exports (lossy -> XSD-valid only).
$PY Database_Integration/python/import_fundsxml.py mx.db "$MX"
$PY Database_Integration/python/export_fundsxml.py mx.db FUNDSXML_FILE_1 mx.xml
xmllint --noout --nonet --schema "$XSD" mx.xml
echo "multi-fund import+export equivalent in python/java/javascript/csharp"
- name: Large-file - streaming aggregate / split / delta (constant memory)
run: |
set -e
PY=.venv/bin/python
P=Large_File_Processing/python
$PY $P/make_large_sample.py big.xml 30000
xmllint --noout --nonet --schema ci-schemas/4.2.9/FundsXML.xsd big.xml
$PY $P/stream_aggregate.py big.xml | tee agg.txt
grep -q '^positions : 30000$' agg.txt
grep -q '^sum value (EUR): 30000000.00$' agg.txt
# Java StreamAggregate (pure JDK) under a small heap proves the
# constant-memory claim; MAVEN_OPTS bounds the in-process JVM.
MAVEN_OPTS=-Xmx64m ./mvnw -q -B -pl Large_File_Processing/java exec:java \
-Dexec.args="big.xml" | tee aggj.txt
grep -q '^positions : 30000$' aggj.txt
$PY $P/split.py big.xml chunks/ 10000
test "$(ls chunks/chunk-*.xml | wc -l)" = "3"
xmllint --noout --nonet --schema ci-schemas/4.2.9/FundsXML.xsd chunks/chunk-0001.xml
$PY $P/delta_diff.py big.xml big.xml # identical -> exit 0
- name: Data binding / JSON - round-trip + native Java binding
run: |
set -e
PY=.venv/bin/python
SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml
$PY Data_Binding_JSON/python/fundsxml_json.py roundtrip "$SRC" rj.xml
xmllint --noout --nonet --schema ci-schemas/4.2.9/FundsXML.xsd rj.xml
# Multi-fund JSON round-trip is lossless -> must be xml_equiv-equal.
MF=FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml
$PY Data_Binding_JSON/python/fundsxml_json.py roundtrip "$MF" rjm.xml
xmllint --noout --nonet --schema ci-schemas/4.2.9/FundsXML.xsd rjm.xml
$PY Database_Integration/tools/xml_equiv.py "$MF" rjm.xml
$PY - "$SRC" rj.xml <<'PY'
import re, sys
o, r = (open(p).read() for p in sys.argv[1:3])
nav = lambda x: re.search(r'<TotalNetAssetValue>\s*<Amount ccy="EUR">([0-9.]+)', x).group(1)
npos = lambda x: len(re.findall(r'<Position>', x))
assert abs(float(nav(o)) - float(nav(r))) < 0.01, (nav(o), nav(r))
assert npos(o) == npos(r), (npos(o), npos(r))
print("JSON round-trip preserved:", nav(r), npos(r))
PY
./mvnw -q -B -pl Data_Binding_JSON/java exec:java -Dexec.args="$SRC" | tee nb.txt
grep -q '^Positions : 21$' nb.txt
- name: Regression - legacy XSLT 1.0 report still runs
run: |
xsltproc XSLT_DataQuality_Checks/Enhanced_Check/FundsXML_CompleteDQReport_HTML.xsl \
FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml > out_enh.html
test -s out_enh.html
# =======================================================================
# Windows smoke — proves the examples are genuinely cross-platform and
# standalone (the whole point of this work): every stack runs with no bash,
# no prior tool step, on a clean Windows box, taking a schema URL + the XML
# file directly. A focused subset (Python / Java / .NET XSD + one XSLT + a
# DB round-trip + the PowerShell CLI), not the full bash-heavy matrix.
# =======================================================================
windows-smoke:
runs-on: windows-latest
defaults:
run:
shell: pwsh
steps:
- uses: actions/checkout@v6
- uses: actions/setup-java@v5
with:
distribution: temurin
java-version: "21"
- uses: actions/setup-python@v6
with:
python-version: "3.11"
- uses: actions/setup-dotnet@v5
with:
dotnet-version: "8.0"
- name: Python - venv & install (pyproject)
run: |
python -m venv .venv
.venv\Scripts\python -m pip install --quiet --upgrade pip
.venv\Scripts\pip install --quiet -e .
.venv\Scripts\python -c "import lxml, saxonche; print('python deps OK')"
- name: Python - XSD validation (remote schema URL on Windows)
run: |
$src = "FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml"
$xsd = "https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd"
.venv\Scripts\python XSD_Validation/python/validate.py $xsd $src
if ($LASTEXITCODE -ne 0) { throw "positive failed" }
.venv\Scripts\python XSD_Validation/python/validate.py $xsd tests/fixtures/invalid/xsd-invalid_Positions.xml
if ($LASTEXITCODE -eq 0) { throw "negative unexpectedly validated (Python)" }
Write-Host "Python OK on Windows"
exit 0 # reset $LASTEXITCODE (the negative case left it at 1)
- name: Java - XSD validate + one XSLT (Maven Wrapper, mvnw.cmd)
run: |
$src = "FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml"
$xsd = "https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd"
.\mvnw.cmd -q -B -pl XSD_Validation/java compile exec:java "-Dexec.args=$xsd $src"
if ($LASTEXITCODE -ne 0) { throw "Java XSD positive failed" }
.\mvnw.cmd -q -B -pl XSD_Validation/java exec:java "-Dexec.args=$xsd tests/fixtures/invalid/xsd-invalid_Positions.xml"
if ($LASTEXITCODE -eq 0) { throw "Java negative unexpectedly validated" }
.\mvnw.cmd -q -B -pl XSLT_Transformations/invocation compile exec:java "-Dexec.args=XSLT_Transformations/CSV_Export/positions_csv.xslt $src out_pos.csv"
if (-not (Test-Path out_pos.csv)) { throw "XSLT produced no output" }
Write-Host "Java OK on Windows"
exit 0 # reset $LASTEXITCODE (the negative case left it at 1)
- name: .NET - XSD validation (remote schema URL on Windows)
run: |
$src = "FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml"
$xsd = "https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd"
dotnet run --project XSD_Validation/dotnet -- $xsd $src
if ($LASTEXITCODE -ne 0) { throw ".NET XSD positive failed" }
dotnet run --project XSD_Validation/dotnet -- $xsd tests/fixtures/invalid/xsd-invalid_Positions.xml
if ($LASTEXITCODE -eq 0) { throw ".NET negative unexpectedly validated" }
Write-Host ".NET OK on Windows"
exit 0 # reset $LASTEXITCODE (the negative case left it at 1)
- name: CLI - validate.ps1 (remote URL, .NET fallback when no xmllint)
run: |
$xsd = "https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd"
pwsh XSD_Validation/cli/validate.ps1 $xsd FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml
if ($LASTEXITCODE -ne 0) { throw "validate.ps1 positive failed" }
pwsh XSD_Validation/cli/validate.ps1 $xsd tests/fixtures/invalid/xsd-invalid_Positions.xml
if ($LASTEXITCODE -eq 0) { throw "validate.ps1 negative unexpectedly validated" }
Write-Host "PowerShell CLI OK on Windows"
exit 0 # reset $LASTEXITCODE (the negative case left it at 1)
- name: DB round-trip (Python) on Windows
run: |
$fx = "FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml"
.venv\Scripts\python Database_Integration/python/import_fundsxml.py fx.db $fx
if ($LASTEXITCODE -ne 0) { throw "import failed" }
.venv\Scripts\python Database_Integration/python/export_fundsxml.py fx.db FUNDSXML_MULTI_1 out.xml
if ($LASTEXITCODE -ne 0) { throw "export failed" }
.venv\Scripts\python Database_Integration/tools/xml_equiv.py $fx out.xml
if ($LASTEXITCODE -ne 0) { throw "round-trip not equivalent" }
Write-Host "DB round-trip OK on Windows"
exit 0