Merge pull request #6 from fundsxml/feature/enterprise-showcase-phase6 #18
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: FundsXML CI | |
| # Validates every committed sample against the official released XSD and the | |
| # Schematron ruleset, runs the XSLT transforms as a smoke test, and asserts the | |
| # negative fixtures still fail. Positive samples must stay green. | |
| # | |
| # Security note: no untrusted GitHub event data (issue/PR/commit text) is used | |
| # in any run: step, so there is no workflow-injection surface here. | |
| on: | |
| push: | |
| branches: ["**"] | |
| pull_request: | |
| workflow_dispatch: | |
| jobs: | |
| validate: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install tooling | |
| run: sudo apt-get update -qq && sudo apt-get install -y -qq libxml2-utils xsltproc | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: "21" | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Fetch official schemas (4.2.9, 4.1.0, 4.0.0) | |
| run: | | |
| for v in 4.2.9 4.1.0 4.0.0; do tools/fetch-schema.sh "$v"; done | |
| - name: XSD - all positive samples must validate | |
| run: | | |
| set -e | |
| declare -A S=( | |
| ["4.2.9"]="positions/Mixed-Fund_Positions transactions/Fund_Transactions documents/Fund_Documents regulatory/EFT_Regulatory signed/Signed_Fund_Skeleton" | |
| ["4.1.0"]="positions/Equity-Fund_Positions" | |
| ["4.0.0"]="positions/Equity-Fund_Positions" | |
| ) | |
| for v in "${!S[@]}"; do | |
| for f in ${S[$v]}; do | |
| echo "XSD $v/$f" | |
| xmllint --noout --nonet --schema ".schema-cache/$v/FundsXML.xsd" \ | |
| "FundsXML_Files/$v/$f.xml" | |
| done | |
| done | |
| - name: XSD - negative fixture must FAIL | |
| run: | | |
| if xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd \ | |
| tests/fixtures/invalid/xsd-invalid_Positions.xml 2>/dev/null; then | |
| echo "::error::xsd-invalid fixture unexpectedly validated"; exit 1 | |
| fi | |
| echo "xsd-invalid correctly rejected" | |
| - name: Fetch Java toolchain (Saxon + SchXslt) | |
| run: tools/fetch-tools.sh | |
| - name: Schematron - canonical sample passes, fixture fails | |
| run: | | |
| RS=Schematron_DataQuality_Checks/Basic_Checks/invocation/run-schematron.sh | |
| "$RS" FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml | |
| if "$RS" tests/fixtures/invalid/schematron-invalid_Positions.xml; then | |
| echo "::error::schematron-invalid fixture unexpectedly passed"; exit 1 | |
| fi | |
| echo "schematron-invalid correctly failed" | |
| - name: Smoke - XSLT transforms produce non-empty output | |
| run: | | |
| set -e | |
| RT=XSLT_Transformations/invocation/run-transform.sh | |
| SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml | |
| "$RT" XSLT_DataQuality_Checks/Custom_Internal_Checks/custom_internal_checks.xslt "$SRC" out_dq.html | |
| "$RT" XSLT_Transformations/Factsheet/factsheet_html.xslt "$SRC" out_fs.html | |
| "$RT" XSLT_Transformations/Factsheet/factsheet_fo.xslt "$SRC" out_fs.fo | |
| "$RT" XSLT_Transformations/CSV_Export/positions_csv.xslt "$SRC" out_pos.csv | |
| xmllint --noout out_fs.fo | |
| for f in out_dq.html out_fs.html out_pos.csv; do | |
| test -s "$f" || { echo "::error::$f is empty"; exit 1; } | |
| done | |
| test "$(wc -l < out_pos.csv)" -ge 2 | |
| - name: Smoke - XQuery examples run and produce XML | |
| run: | | |
| set -e | |
| RQ=XQuery_Examples/invocation/run-xquery.sh | |
| SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml | |
| "$RQ" XQuery_Examples/aggregate-by-assettype.xq "$SRC" out_agg.xml | |
| "$RQ" XQuery_Examples/top-holdings.xq "$SRC" out_top.xml n=5 | |
| "$RQ" XQuery_Examples/fund-summary.xq "$SRC" out_sum.xml | |
| "$RQ" XQuery_Examples/look-through.xq "$SRC" out_lt.xml | |
| for f in out_agg.xml out_top.xml out_sum.xml out_lt.xml; do | |
| xmllint --noout "$f" | |
| done | |
| test "$(xmllint --xpath 'count(//Holding)' out_top.xml)" = "5" | |
| - name: XML signature - Apache Santuario sign/verify/tamper roundtrip | |
| run: | | |
| set -e | |
| XML_Signature/generate-test-key.sh | |
| CP=.lib/xmlsec-4.0.4.jar:.lib/commons-codec-1.18.0.jar:.lib/slf4j-api-2.0.17.jar:.lib/slf4j-nop-2.0.17.jar | |
| javac -cp "$CP" -d /tmp/sig \ | |
| XML_Signature/java/SignFundsXml.java XML_Signature/java/VerifyFundsXml.java | |
| SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml | |
| java -cp "$CP:/tmp/sig" SignFundsXml "$SRC" signed.xml \ | |
| XML_Signature/keys/test-signing.p12 changeit fundsxml | |
| java -cp "$CP:/tmp/sig" VerifyFundsXml signed.xml \ | |
| XML_Signature/keys/test-signing-cert.pem | |
| xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd signed.xml | |
| sed 's/<TotalPercentage>8.33</<TotalPercentage>9.33</' signed.xml > tampered.xml | |
| if java -cp "$CP:/tmp/sig" VerifyFundsXml tampered.xml \ | |
| XML_Signature/keys/test-signing-cert.pem; then | |
| echo "::error::tampered file unexpectedly verified"; exit 1 | |
| fi | |
| echo "tamper correctly detected" | |
| - name: DB integration - FundsXML -> SQLite -> FundsXML round-trip | |
| run: | | |
| set -e | |
| python3 -m pip install --quiet lxml | |
| SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml | |
| python3 Database_Integration/python/fundsxml_db.py roundtrip "$SRC" regen.xml | |
| xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd regen.xml | |
| python3 - "$SRC" regen.xml <<'PY' | |
| import re, sys | |
| o, r = (open(p).read() for p in sys.argv[1:3]) | |
| nav = lambda x: re.search(r'<TotalNetAssetValue>\s*<Amount ccy="EUR">([0-9.]+)', x).group(1) | |
| npos = lambda x: len(re.findall(r'<Position>', x)) | |
| spct = lambda x: round(sum(float(v) for v in re.findall(r'<TotalPercentage>([0-9.]+)', x)), 2) | |
| assert abs(float(nav(o)) - float(nav(r))) < 0.01, (nav(o), nav(r)) | |
| assert npos(o) == npos(r), (npos(o), npos(r)) | |
| assert abs(spct(o) - spct(r)) < 0.01, (spct(o), spct(r)) | |
| print("round-trip figures preserved:", nav(r), npos(r), spct(r)) | |
| PY | |
| - name: Large-file - streaming aggregate / split / delta (constant memory) | |
| run: | | |
| set -e | |
| python3 -m pip install --quiet lxml | |
| P=Large_File_Processing/python | |
| python3 $P/make_large_sample.py big.xml 30000 | |
| xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd big.xml | |
| python3 $P/stream_aggregate.py big.xml | tee agg.txt | |
| grep -q '^positions : 30000$' agg.txt | |
| grep -q '^sum value (EUR): 30000000.00$' agg.txt | |
| javac -d /tmp/lf Large_File_Processing/java/StreamAggregate.java | |
| java -Xmx64m -cp /tmp/lf StreamAggregate big.xml | tee aggj.txt | |
| grep -q '^positions : 30000$' aggj.txt | |
| python3 $P/split.py big.xml chunks/ 10000 | |
| test "$(ls chunks/chunk-*.xml | wc -l)" = "3" | |
| xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd chunks/chunk-0001.xml | |
| python3 $P/delta_diff.py big.xml big.xml # identical -> exit 0 | |
| - name: Data binding / JSON - round-trip + native Java binding | |
| run: | | |
| set -e | |
| python3 -m pip install --quiet lxml | |
| SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml | |
| python3 Data_Binding_JSON/python/fundsxml_json.py roundtrip "$SRC" rj.xml | |
| xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd rj.xml | |
| python3 - "$SRC" rj.xml <<'PY' | |
| import re, sys | |
| o, r = (open(p).read() for p in sys.argv[1:3]) | |
| nav = lambda x: re.search(r'<TotalNetAssetValue>\s*<Amount ccy="EUR">([0-9.]+)', x).group(1) | |
| npos = lambda x: len(re.findall(r'<Position>', x)) | |
| assert abs(float(nav(o)) - float(nav(r))) < 0.01, (nav(o), nav(r)) | |
| assert npos(o) == npos(r), (npos(o), npos(r)) | |
| print("JSON round-trip preserved:", nav(r), npos(r)) | |
| PY | |
| javac -d /tmp/nb Data_Binding_JSON/java/NativeBinding.java | |
| java -cp /tmp/nb NativeBinding "$SRC" | tee nb.txt | |
| grep -q '^Positions : 21$' nb.txt | |
| - name: Regression - legacy XSLT 1.0 report still runs | |
| run: | | |
| xsltproc XSLT_DataQuality_Checks/Enhanced_Check/FundsXML_CompleteDQReport_HTML.xsl \ | |
| FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml > out_enh.html | |
| test -s out_enh.html |