diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4b087b..b4de66d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,6 +100,45 @@ jobs: done test "$(xmllint --xpath 'count(//Holding)' out_top.xml)" = "5" + - name: XML signature - Apache Santuario sign/verify/tamper roundtrip + run: | + set -e + XML_Signature/generate-test-key.sh + CP=.lib/xmlsec-4.0.4.jar:.lib/commons-codec-1.18.0.jar:.lib/slf4j-api-2.0.17.jar:.lib/slf4j-nop-2.0.17.jar + javac -cp "$CP" -d /tmp/sig \ + XML_Signature/java/SignFundsXml.java XML_Signature/java/VerifyFundsXml.java + SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml + java -cp "$CP:/tmp/sig" SignFundsXml "$SRC" signed.xml \ + XML_Signature/keys/test-signing.p12 changeit fundsxml + java -cp "$CP:/tmp/sig" VerifyFundsXml signed.xml \ + XML_Signature/keys/test-signing-cert.pem + xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd signed.xml + sed 's/8.339.33 tampered.xml + if java -cp "$CP:/tmp/sig" VerifyFundsXml tampered.xml \ + XML_Signature/keys/test-signing-cert.pem; then + echo "::error::tampered file unexpectedly verified"; exit 1 + fi + echo "tamper correctly detected" + + - name: DB integration - FundsXML -> SQLite -> FundsXML round-trip + run: | + set -e + python3 -m pip install --quiet lxml + SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml + python3 Database_Integration/python/fundsxml_db.py roundtrip "$SRC" regen.xml + xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd regen.xml + python3 - "$SRC" regen.xml <<'PY' + import re, sys + o, r = (open(p).read() for p in sys.argv[1:3]) + nav = lambda x: re.search(r'\s*([0-9.]+)', x).group(1) + npos = lambda x: len(re.findall(r'', x)) + spct = lambda x: round(sum(float(v) for v in re.findall(r'([0-9.]+)', x)), 2) + assert abs(float(nav(o)) - float(nav(r))) < 0.01, (nav(o), nav(r)) + assert npos(o) == npos(r), (npos(o), npos(r)) + assert abs(spct(o) - spct(r)) < 0.01, (spct(o), spct(r)) + print("round-trip figures preserved:", nav(r), npos(r), spct(r)) + PY + - name: Regression - legacy XSLT 1.0 report still runs run: | xsltproc XSLT_DataQuality_Checks/Enhanced_Check/FundsXML_CompleteDQReport_HTML.xsl \ diff --git a/.gitignore b/.gitignore index e863448..5bf9d08 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,10 @@ CLAUDE.md # dependencies from Maven Central, not repo source. .lib/ +# Throwaway signing keys (XML_Signature/generate-test-key.sh) — never commit +# private keys, even demo ones. +XML_Signature/keys/ + # Generated reports / outputs *.report.html report.html @@ -18,4 +22,7 @@ out_*.html out_*.fo out_*.csv out_*.xml +regen.xml +regenerated.xml +*.db *.svrl diff --git a/Database_Integration/README.md b/Database_Integration/README.md new file mode 100644 index 0000000..b30f2fd --- /dev/null +++ b/Database_Integration/README.md @@ -0,0 +1,48 @@ +# Database Integration + +![python](https://img.shields.io/badge/SQLite%20round--trip-verified-brightgreen) ![sql](https://img.shields.io/badge/Oracle%2FSQLServer%2FPostgres-code--reference-blue) + +FundsXML ⇄ relational database, both directions. Per project scope **no +database is provisioned** (no Docker): the Oracle / SQL Server / PostgreSQL +files are **code references**, and the **SQLite Python implementation is the +runnable, verified reference** (stdlib + lxml only) that exercises the full +round-trip and produces XSD-valid FundsXML. + +## Layout + +| Path | What | +|------|------| +| [`ddl/schema.sql`](ddl/schema.sql) | Shared relational model (fund, share_class, asset, position) | +| [`load_from_fundsxml/`](load_from_fundsxml/) | FundsXML → DB: `postgres.sql` (XMLTABLE), `oracle.sql` (XMLType + XMLTABLE), `sqlserver.sql` (`.nodes()`/`.value()`) | +| [`generate_fundsxml/`](generate_fundsxml/) | DB → FundsXML: SQL/XML publishing (`xmlelement`/`xmlagg`, `XMLElement`/`XMLAgg`, `FOR XML PATH`) | +| [`python/fundsxml_db.py`](python/fundsxml_db.py) | **Runnable** SQLite reference: `init` / `load` / `generate` / `roundtrip` | + +The relational model is keyed by `ControlData/UniqueDocumentID` so multiple +documents coexist. FundsXML 4.x has no XML namespace — all XPath/queries use +bare element names. + +## Runnable round-trip (verified) + +```bash +python3 Database_Integration/python/fundsxml_db.py roundtrip \ + FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml regenerated.xml + +tools/fetch-schema.sh 4.2.9 +xmllint --noout --schema .schema-cache/4.2.9/FundsXML.xsd regenerated.xml +``` + +Verified for the 4.2.9 positions, 4.1.0 and 4.0.0 positions, and 4.2.9 +transactions samples: each loads into SQLite and regenerates **XSD-valid** +FundsXML with NAV, position count and percentage sum preserved. + +### Round-trip fidelity + +The model captures fund header, share classes (incl. `TotalAssetValues` / +`SharesOutstanding`), positions (value, %, class + quantity child) and asset +master data. It is intentionally **lossy** for everything outside that core +(issuer details, derivative terms, transactions, regulatory blocks) — the +regenerated file is a faithful *positions* document, normalized to the 4.2.9 +schema (a 4.0.0 input comes back without `ControlData/Version`, which is valid). + +The three enterprise-DB SQL files mirror the same shred/publish logic; only the +SQLite path is executed here. CI runs the round-trip on every push. diff --git a/Database_Integration/ddl/schema.sql b/Database_Integration/ddl/schema.sql new file mode 100644 index 0000000..29e0d83 --- /dev/null +++ b/Database_Integration/ddl/schema.sql @@ -0,0 +1,65 @@ +-- Shared relational model for FundsXML positions data. +-- +-- Portable ANSI-ish DDL (works on SQLite/Postgres; see notes for Oracle and +-- SQL Server type tweaks). One row set per source document, keyed by the +-- ControlData/UniqueDocumentID so multiple files can coexist. +-- +-- Faithful enough to round-trip the canonical positions sample back to +-- XSD-valid FundsXML (fund + share classes + positions + asset master data). + +CREATE TABLE fund ( + document_id VARCHAR(128) NOT NULL, + lei VARCHAR(20), + official_name VARCHAR(256) NOT NULL, + currency CHAR(3) NOT NULL, + content_date DATE, + nav_date DATE, + total_nav DECIMAL(20,2) NOT NULL, + fxml_version VARCHAR(16), -- absent for 4.0.0 + PRIMARY KEY (document_id) +); + +CREATE TABLE share_class ( + document_id VARCHAR(128) NOT NULL, + isin CHAR(12) NOT NULL, + official_name VARCHAR(256), + currency CHAR(3) NOT NULL, + nav_price DECIMAL(20,6), + nav_fund_ccy DECIMAL(20,2), -- NAV in fund currency + shares_outstanding DECIMAL(28,6), + PRIMARY KEY (document_id, isin), + FOREIGN KEY (document_id) REFERENCES fund (document_id) +); + +CREATE TABLE asset ( + document_id VARCHAR(128) NOT NULL, + unique_id VARCHAR(256) NOT NULL, + isin CHAR(12), + name VARCHAR(256), + asset_type CHAR(2), -- EQ, BO, SC, ... + currency CHAR(3), + country CHAR(2), + PRIMARY KEY (document_id, unique_id), + FOREIGN KEY (document_id) REFERENCES fund (document_id) +); + +CREATE TABLE position ( + document_id VARCHAR(128) NOT NULL, + unique_id VARCHAR(256) NOT NULL, -- joins to asset.unique_id + isin CHAR(12), + currency CHAR(3), + value_fund_ccy DECIMAL(20,2) NOT NULL, + percentage DECIMAL(9,4) NOT NULL, + kind VARCHAR(16), -- Position class element: + -- Equity/Bond/ShareClass/... + kind_qty DECIMAL(28,6), -- its quantity child + -- (Units/Nominal/Shares/Contracts) + PRIMARY KEY (document_id, unique_id), + FOREIGN KEY (document_id) REFERENCES fund (document_id) +); + +-- Dialect notes +-- Oracle : VARCHAR(n) -> VARCHAR2(n); DECIMAL -> NUMBER; CHAR ok. +-- An XMLType staging column is shown in load_from_fundsxml. +-- SQL Server : VARCHAR ok; DECIMAL ok; use an XML column for staging. +-- Postgres : as-is; a native `xml` staging column is used for xmltable. diff --git a/Database_Integration/generate_fundsxml/oracle.sql b/Database_Integration/generate_fundsxml/oracle.sql new file mode 100644 index 0000000..c347617 --- /dev/null +++ b/Database_Integration/generate_fundsxml/oracle.sql @@ -0,0 +1,60 @@ +-- Oracle -> FundsXML (code reference; no DB is provisioned in this repo). +-- +-- SQL/XML publishing: XMLElement / XMLAgg / XMLAttributes. Same FundsXML 4.2.9 +-- positions shape as the Python reference. Bind :doc = fund.document_id. + +SELECT XMLElement("FundsXML4", + XMLAttributes( + 'http://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd' + AS "xsi:noNamespaceSchemaLocation"), + XMLElement("ControlData", + XMLElement("UniqueDocumentID", f.document_id), + XMLElement("DocumentGenerated", '2025-10-02T00:00:00'), + CASE WHEN f.fxml_version IS NOT NULL + THEN XMLElement("Version", f.fxml_version) END, + XMLElement("ContentDate", TO_CHAR(f.content_date,'YYYY-MM-DD')), + XMLElement("DataSupplier", + XMLElement("SystemCountry",'AT'), + XMLElement("Short",'EURAM'), + XMLElement("Name",'Erste Asset Management GmbH'), + XMLElement("Type",'Asset Manager')), + XMLElement("DataOperation",'INITIAL')), + XMLElement("Funds", + XMLElement("Fund", + XMLElement("Identifiers", XMLElement("LEI", f.lei)), + XMLElement("Names", XMLElement("OfficialName", f.official_name)), + XMLElement("Currency", f.currency), + XMLElement("SingleFundFlag",'true'), + XMLElement("FundDynamicData", + XMLElement("TotalAssetValues", + XMLElement("TotalAssetValue", + XMLElement("NavDate", TO_CHAR(f.nav_date,'YYYY-MM-DD')), + XMLElement("TotalAssetNature",'OFFICIAL'), + XMLElement("TotalNetAssetValue", + XMLElement("Amount", + XMLAttributes(f.currency AS "ccy"), + TO_CHAR(f.total_nav,'FM999999999990.00'))))), + XMLElement("Portfolios", + XMLElement("Portfolio", + XMLElement("NavDate", TO_CHAR(f.nav_date,'YYYY-MM-DD')), + XMLElement("Positions", + (SELECT XMLAgg( + XMLElement("Position", + XMLElement("UniqueID", p.unique_id), + XMLElement("Currency", p.currency), + XMLElement("TotalValue", + XMLElement("Amount", + XMLAttributes(f.currency AS "ccy"), + TO_CHAR(p.value_fund_ccy,'FM999999999990.00'))), + XMLElement("TotalPercentage", + TO_CHAR(p.percentage,'FM990.00')), + XMLElement(EVALNAME p.kind)) + ORDER BY p.unique_id) + FROM position p + WHERE p.document_id = f.document_id))))))) + ).getClobVal() +FROM fund f +WHERE f.document_id = :doc; + +-- As with the Postgres example, add the quantity child per kind +-- (position.kind_qty) to keep Equity/Bond/ShareClass/... schema-valid. diff --git a/Database_Integration/generate_fundsxml/postgres.sql b/Database_Integration/generate_fundsxml/postgres.sql new file mode 100644 index 0000000..b7d2833 --- /dev/null +++ b/Database_Integration/generate_fundsxml/postgres.sql @@ -0,0 +1,64 @@ +-- PostgreSQL -> FundsXML (code reference; no DB is provisioned in this repo). +-- +-- SQL/XML publishing with xmlelement/xmlagg/xmlattributes. Produces the same +-- FundsXML 4.2.9 positions shape the Python reference emits (XSD-valid). +-- Parameter :doc = fund.document_id. + +SELECT xmlelement(name "FundsXML4", + xmlattributes('http://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd' + AS "xsi:noNamespaceSchemaLocation"), + xmlelement(name "ControlData", + xmlelement(name "UniqueDocumentID", f.document_id), + xmlelement(name "DocumentGenerated", '2025-10-02T00:00:00'), + CASE WHEN f.fxml_version IS NOT NULL + THEN xmlelement(name "Version", f.fxml_version) END, + xmlelement(name "ContentDate", f.content_date), + xmlelement(name "DataSupplier", + xmlelement(name "SystemCountry",'AT'), + xmlelement(name "Short",'EURAM'), + xmlelement(name "Name",'Erste Asset Management GmbH'), + xmlelement(name "Type",'Asset Manager')), + xmlelement(name "DataOperation",'INITIAL')), + xmlelement(name "Funds", + xmlelement(name "Fund", + xmlelement(name "Identifiers", + xmlelement(name "LEI", f.lei)), + xmlelement(name "Names", + xmlelement(name "OfficialName", f.official_name)), + xmlelement(name "Currency", f.currency), + xmlelement(name "SingleFundFlag",'true'), + xmlelement(name "FundDynamicData", + xmlelement(name "TotalAssetValues", + xmlelement(name "TotalAssetValue", + xmlelement(name "NavDate", f.nav_date), + xmlelement(name "TotalAssetNature",'OFFICIAL'), + xmlelement(name "TotalNetAssetValue", + xmlelement(name "Amount", + xmlattributes(f.currency AS "ccy"), + to_char(f.total_nav,'FM999999999990.00'))))), + xmlelement(name "Portfolios", + xmlelement(name "Portfolio", + xmlelement(name "NavDate", f.nav_date), + xmlelement(name "Positions", + (SELECT xmlagg( + xmlelement(name "Position", + xmlelement(name "UniqueID", p.unique_id), + CASE WHEN p.isin IS NOT NULL THEN + xmlelement(name "Identifiers", + xmlelement(name "ISIN", p.isin)) END, + xmlelement(name "Currency", p.currency), + xmlelement(name "TotalValue", + xmlelement(name "Amount", + xmlattributes(f.currency AS "ccy"), + to_char(p.value_fund_ccy,'FM999999999990.00'))), + xmlelement(name "TotalPercentage", + to_char(p.percentage,'FM990.00')), + xmlelement(name p.kind)) ORDER BY p.unique_id) + FROM position p WHERE p.document_id = f.document_id)))))) + ) +FROM fund f +WHERE f.document_id = :doc; + +-- For brevity the position class element is emitted empty; add the quantity +-- child (Units/Nominal/Shares/Contracts from position.kind_qty) exactly as the +-- Python reference does to keep Equity/Bond/ShareClass/... schema-valid. diff --git a/Database_Integration/generate_fundsxml/sqlserver.sql b/Database_Integration/generate_fundsxml/sqlserver.sql new file mode 100644 index 0000000..bb607c1 --- /dev/null +++ b/Database_Integration/generate_fundsxml/sqlserver.sql @@ -0,0 +1,51 @@ +-- SQL Server -> FundsXML (code reference; no DB is provisioned in this repo). +-- +-- FOR XML PATH with nested subqueries. Same FundsXML 4.2.9 positions shape as +-- the Python reference. @doc = fund.document_id. + +DECLARE @doc varchar(128) = 'FUNDSXML_FILE_1'; + +SELECT + 'http://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd' + AS [@xsi:noNamespaceSchemaLocation], + f.document_id AS [ControlData/UniqueDocumentID], + '2025-10-02T00:00:00' AS [ControlData/DocumentGenerated], + f.fxml_version AS [ControlData/Version], + CONVERT(varchar(10), f.content_date, 23) AS [ControlData/ContentDate], + 'AT' AS [ControlData/DataSupplier/SystemCountry], + 'EURAM' AS [ControlData/DataSupplier/Short], + 'Erste Asset Management GmbH' AS [ControlData/DataSupplier/Name], + 'Asset Manager' AS [ControlData/DataSupplier/Type], + 'INITIAL' AS [ControlData/DataOperation], + f.lei AS [Funds/Fund/Identifiers/LEI], + f.official_name AS [Funds/Fund/Names/OfficialName], + f.currency AS [Funds/Fund/Currency], + 'true' AS [Funds/Fund/SingleFundFlag], + CONVERT(varchar(10), f.nav_date, 23) + AS [Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/NavDate], + 'OFFICIAL' + AS [Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/TotalAssetNature], + f.currency + AS [Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount/@ccy], + CONVERT(varchar(32), f.total_nav) + AS [Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount], + (SELECT + p.unique_id AS [UniqueID], + p.currency AS [Currency], + f.currency AS [TotalValue/Amount/@ccy], + CONVERT(varchar(32), p.value_fund_ccy) AS [TotalValue/Amount], + CONVERT(varchar(16), p.percentage) AS [TotalPercentage] + -- plus the kind element (+ quantity child) — see note below + FROM position p + WHERE p.document_id = f.document_id + ORDER BY p.unique_id + FOR XML PATH('Position'), TYPE) + AS [Funds/Fund/FundDynamicData/Portfolios/Portfolio/Positions] +FROM fund f +WHERE f.document_id = @doc +FOR XML PATH('FundsXML4'); + +-- The position class element (Equity/Bond/.../with its Units/Nominal/Shares/ +-- Contracts child from position.kind_qty) is added with a correlated +-- FOR XML PATH subquery per row — emitted dynamically because the element name +-- itself varies; see the Python reference for the exact mapping. diff --git a/Database_Integration/load_from_fundsxml/oracle.sql b/Database_Integration/load_from_fundsxml/oracle.sql new file mode 100644 index 0000000..6b7e8a8 --- /dev/null +++ b/Database_Integration/load_from_fundsxml/oracle.sql @@ -0,0 +1,48 @@ +-- FundsXML -> Oracle (code reference; no DB is provisioned in this repo). +-- +-- Strategy: stage in an XMLType column, shred with XMLTABLE. FundsXML 4.x has +-- no namespace. Schema: ../ddl/schema.sql with VARCHAR2/NUMBER types. + +CREATE TABLE fundsxml_stage ( + document_id VARCHAR2(128) PRIMARY KEY, + doc XMLTYPE NOT NULL +); + +-- Load (SQL*Plus / SQLcl): use a BFILE or :bind variable, e.g. +-- INSERT INTO fundsxml_stage(document_id, doc) +-- VALUES ( :doc.extract('/FundsXML4/ControlData/UniqueDocumentID/text()') +-- .getStringVal(), XMLTYPE(:doc) ); + +INSERT INTO fund (document_id, lei, official_name, currency, + content_date, nav_date, total_nav, fxml_version) +SELECT s.document_id, f.lei, f.official_name, f.currency, + TO_DATE(f.content_date,'YYYY-MM-DD'), + TO_DATE(f.nav_date,'YYYY-MM-DD'), f.total_nav, f.fxml_version +FROM fundsxml_stage s, + XMLTABLE('/FundsXML4' PASSING s.doc COLUMNS + lei VARCHAR2(20) PATH 'Funds/Fund/Identifiers/LEI', + official_name VARCHAR2(256) PATH 'Funds/Fund/Names/OfficialName', + currency VARCHAR2(3) PATH 'Funds/Fund/Currency', + content_date VARCHAR2(10) PATH 'ControlData/ContentDate', + fxml_version VARCHAR2(16) PATH 'ControlData/Version', + nav_date VARCHAR2(10) PATH 'Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/NavDate', + total_nav NUMBER PATH 'Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount[@ccy=/FundsXML4/Funds/Fund/Currency]' + ) f; + +INSERT INTO position (document_id, unique_id, isin, currency, + value_fund_ccy, percentage, kind, kind_qty) +SELECT s.document_id, p.unique_id, p.isin, p.currency, + p.value_fund_ccy, p.percentage, p.kind, p.kind_qty +FROM fundsxml_stage s, + XMLTABLE('/FundsXML4/Funds/Fund/FundDynamicData/Portfolios/Portfolio/Positions/Position' + PASSING s.doc COLUMNS + unique_id VARCHAR2(256) PATH 'UniqueID', + isin VARCHAR2(12) PATH 'Identifiers/ISIN', + currency VARCHAR2(3) PATH 'Currency', + value_fund_ccy NUMBER PATH 'TotalValue/Amount[1]', + percentage NUMBER PATH 'TotalPercentage', + kind VARCHAR2(16) PATH 'name(*[local-name()=("Equity","Bond","ShareClass","Warrant","Certificate","Option","Future","FXForward","Swap","Repo","RealEstate","CallMoney")][1])', + kind_qty NUMBER PATH '(Equity/Units|Bond/Nominal|ShareClass/Shares|Warrant/Units|Certificate/Units|Option/Contracts|Future/Contracts)[1]' + ) p; + +-- asset / share_class: same XMLTABLE pattern over their node sets. diff --git a/Database_Integration/load_from_fundsxml/postgres.sql b/Database_Integration/load_from_fundsxml/postgres.sql new file mode 100644 index 0000000..efb6d63 --- /dev/null +++ b/Database_Integration/load_from_fundsxml/postgres.sql @@ -0,0 +1,56 @@ +-- FundsXML -> PostgreSQL (code reference; no DB is provisioned in this repo). +-- +-- Strategy: stage the document in a native `xml` column, then shred with +-- XMLTABLE. FundsXML 4.x has no namespace, so XPath uses bare element names. +-- Schema: ../ddl/schema.sql (run that first, plus the staging table below). + +CREATE TABLE IF NOT EXISTS fundsxml_stage ( + document_id text PRIMARY KEY, + doc xml NOT NULL +); + +-- Load the file into the stage (psql): +-- \set content `cat FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml` +-- INSERT INTO fundsxml_stage +-- SELECT (xpath('/FundsXML4/ControlData/UniqueDocumentID/text()', +-- x.doc))[1]::text, x.doc +-- FROM (SELECT :'content'::xml AS doc) x; + +-- fund ----------------------------------------------------------------------- +INSERT INTO fund (document_id, lei, official_name, currency, + content_date, nav_date, total_nav, fxml_version) +SELECT s.document_id, f.lei, f.official_name, f.currency, + f.content_date, f.nav_date, f.total_nav, f.fxml_version +FROM fundsxml_stage s, + XMLTABLE('/FundsXML4' PASSING s.doc COLUMNS + lei text PATH 'Funds/Fund/Identifiers/LEI', + official_name text PATH 'Funds/Fund/Names/OfficialName', + currency text PATH 'Funds/Fund/Currency', + content_date date PATH 'ControlData/ContentDate', + fxml_version text PATH 'ControlData/Version', + nav_date date PATH 'Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/NavDate', + total_nav numeric(20,2) PATH 'Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount[@ccy=/FundsXML4/Funds/Fund/Currency]' + ) f; + +-- position ------------------------------------------------------------------- +INSERT INTO position (document_id, unique_id, isin, currency, + value_fund_ccy, percentage, kind, kind_qty) +SELECT s.document_id, p.unique_id, p.isin, p.currency, + p.value_fund_ccy, p.percentage, p.kind, p.kind_qty +FROM fundsxml_stage s, + LATERAL (SELECT (xpath('/FundsXML4/Funds/Fund/Currency/text()', + s.doc))[1]::text AS ccy) c, + XMLTABLE('/FundsXML4/Funds/Fund/FundDynamicData/Portfolios/Portfolio/Positions/Position' + PASSING s.doc COLUMNS + unique_id text PATH 'UniqueID', + isin text PATH 'Identifiers/ISIN', + currency text PATH 'Currency', + value_fund_ccy numeric(20,2) PATH 'TotalValue/Amount[1]', + percentage numeric(9,4) PATH 'TotalPercentage', + kind text PATH 'local-name(Equity|Bond|ShareClass|Warrant|Certificate|Option|Future|FXForward|Swap|Repo|RealEstate|CallMoney)', + kind_qty numeric(28,6) PATH '(Equity/Units|Bond/Nominal|ShareClass/Shares|Warrant/Units|Certificate/Units|Option/Contracts|Future/Contracts)[1]' + ) p; + +-- asset and share_class follow the same XMLTABLE pattern over +-- /FundsXML4/AssetMasterData/Asset and +-- /FundsXML4/Funds/Fund/SingleFund/ShareClasses/ShareClass respectively. diff --git a/Database_Integration/load_from_fundsxml/sqlserver.sql b/Database_Integration/load_from_fundsxml/sqlserver.sql new file mode 100644 index 0000000..7dc0d40 --- /dev/null +++ b/Database_Integration/load_from_fundsxml/sqlserver.sql @@ -0,0 +1,47 @@ +-- FundsXML -> SQL Server (code reference; no DB is provisioned in this repo). +-- +-- Strategy: stage in an `xml` column, shred with .nodes()/.value(). FundsXML +-- 4.x has no namespace. Schema: ../ddl/schema.sql. + +CREATE TABLE fundsxml_stage ( + document_id VARCHAR(128) PRIMARY KEY, + doc XML NOT NULL +); + +-- Load: read the file with OPENROWSET(BULK ... SINGLE_CLOB) then +-- INSERT INTO fundsxml_stage(document_id, doc) +-- SELECT x.value('(/FundsXML4/ControlData/UniqueDocumentID)[1]','varchar(128)'), +-- x +-- FROM (SELECT CAST(BulkColumn AS XML) AS x +-- FROM OPENROWSET(BULK '...Mixed-Fund_Positions.xml', +-- SINGLE_CLOB) r) src; + +INSERT INTO fund (document_id, lei, official_name, currency, + content_date, nav_date, total_nav, fxml_version) +SELECT s.document_id, + d.value('(Funds/Fund/Identifiers/LEI)[1]','varchar(20)'), + d.value('(Funds/Fund/Names/OfficialName)[1]','varchar(256)'), + d.value('(Funds/Fund/Currency)[1]','char(3)'), + d.value('(ControlData/ContentDate)[1]','date'), + d.value('(Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/NavDate)[1]','date'), + d.value('(Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount)[1]','decimal(20,2)'), + d.value('(ControlData/Version)[1]','varchar(16)') +FROM fundsxml_stage s +CROSS APPLY s.doc.nodes('/FundsXML4') AS t(d); + +INSERT INTO position (document_id, unique_id, isin, currency, + value_fund_ccy, percentage, kind, kind_qty) +SELECT s.document_id, + p.value('(UniqueID)[1]','varchar(256)'), + p.value('(Identifiers/ISIN)[1]','char(12)'), + p.value('(Currency)[1]','char(3)'), + p.value('(TotalValue/Amount)[1]','decimal(20,2)'), + p.value('(TotalPercentage)[1]','decimal(9,4)'), + p.value('local-name((Equity|Bond|ShareClass|Warrant|Certificate|Option|Future|FXForward|Swap|Repo|RealEstate|CallMoney)[1])','varchar(16)'), + p.value('(Equity/Units|Bond/Nominal|ShareClass/Shares|Warrant/Units|Certificate/Units|Option/Contracts|Future/Contracts)[1]','decimal(28,6)') +FROM fundsxml_stage s +CROSS APPLY s.doc.nodes( + '/FundsXML4/Funds/Fund/FundDynamicData/Portfolios/Portfolio/Positions/Position' +) AS t(p); + +-- asset / share_class: same CROSS APPLY .nodes() / .value() pattern. diff --git a/Database_Integration/python/fundsxml_db.py b/Database_Integration/python/fundsxml_db.py new file mode 100644 index 0000000..6e46c2d --- /dev/null +++ b/Database_Integration/python/fundsxml_db.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +"""FundsXML <-> relational database — runnable reference (SQLite, stdlib). + +The Oracle / SQL Server / Postgres examples in ../load_from_fundsxml/ and +../generate_fundsxml/ are code-only (no DB is provisioned, per project scope). +This SQLite implementation is the *executable* reference: it needs nothing +beyond the Python stdlib + lxml, runs the full FundsXML -> DB -> FundsXML +round-trip, and the regenerated file is XSD-valid. + +Subcommands: + init create the schema (ddl/schema.sql) + load shred a FundsXML file into the tables + generate rebuild XSD-valid FundsXML from the tables + roundtrip load + generate via a temp DB (one shot) + +Relational model: ../ddl/schema.sql. FundsXML 4.x has no XML namespace, so all +XPath uses bare element names. +""" +import sqlite3 +import sys +import tempfile +from pathlib import Path + +from lxml import etree + +# Position class elements (the required child after TotalPercentage). Many are +# valid when empty (FXForward/Swap/Repo/RealEstate/CallMoney in the canonical +# sample); the rest accept an empty element for this round-trip demo. +POSITION_KINDS = {"Equity", "Bond", "ShareClass", "Warrant", "Certificate", + "Option", "Future", "FXForward", "Swap", "Repo", + "RealEstate", "CallMoney", "Account", "Generic"} +# Kinds that require a numeric quantity child -> the child element name. +# The remaining kinds (FXForward/Swap/Repo/RealEstate/CallMoney/...) are +# schema-valid as an empty element. +QTY_ELEM = {"Equity": "Units", "Warrant": "Units", "Certificate": "Units", + "Bond": "Nominal", "ShareClass": "Shares", + "Option": "Contracts", "Future": "Contracts"} + +DDL = Path(__file__).resolve().parents[1] / "ddl" / "schema.sql" +SCHEMA_URL = ("https://github.com/fundsxml/schema/releases/download/" + "4.2.9/FundsXML.xsd") +XSI = "http://www.w3.org/2001/XMLSchema-instance" + + +# ---------------------------------------------------------------- load ------- +def _txt(node, path, default=None): + r = node.xpath(path) + return r[0].text if r and r[0].text is not None else default + + +def load(db: str, xml_path: str) -> str: + doc = etree.parse(xml_path) + cd = doc.xpath("/FundsXML4/ControlData")[0] + fund = doc.xpath("/FundsXML4/Funds/Fund")[0] + doc_id = _txt(cd, "UniqueDocumentID") + ccy = _txt(fund, "Currency") + tav = fund.xpath("FundDynamicData/TotalAssetValues/TotalAssetValue")[0] + nav = tav.xpath(f"TotalNetAssetValue/Amount[@ccy='{ccy}']")[0].text + + con = sqlite3.connect(db) + con.execute("PRAGMA foreign_keys=ON") + con.execute( + "INSERT INTO fund VALUES (?,?,?,?,?,?,?,?)", + (doc_id, _txt(fund, "Identifiers/LEI"), + _txt(fund, "Names/OfficialName"), ccy, + _txt(cd, "ContentDate"), _txt(tav, "NavDate"), + float(nav), _txt(cd, "Version"))) + + for sc in fund.xpath("SingleFund/ShareClasses/ShareClass"): + sccy = _txt(sc, "Currency") + con.execute( + "INSERT INTO share_class VALUES (?,?,?,?,?,?,?)", + (doc_id, _txt(sc, "Identifiers/ISIN"), + _txt(sc, "Names/OfficialName"), sccy, + _num(_txt(sc, "Prices/Price/NavPrice")), + _num(_first(sc, "TotalAssetValues/TotalAssetValue/" + f"TotalNetAssetValue/Amount[@ccy='{ccy}']")), + _num(_txt(sc, "TotalAssetValues/TotalAssetValue/" + "SharesOutstanding")))) + + for a in doc.xpath("/FundsXML4/AssetMasterData/Asset"): + con.execute( + "INSERT INTO asset VALUES (?,?,?,?,?,?,?)", + (doc_id, _txt(a, "UniqueID"), _txt(a, "Identifiers/ISIN"), + _txt(a, "Name"), _txt(a, "AssetType"), + _txt(a, "Currency"), _txt(a, "Country"))) + + for p in fund.xpath("FundDynamicData/Portfolios/Portfolio/Positions/Position"): + kinds = [c.tag for c in p if c.tag in POSITION_KINDS] + kind = kinds[0] if kinds else None + qty = None + if kind in QTY_ELEM: + qty = _num(_txt(p, f"{kind}/{QTY_ELEM[kind]}")) + con.execute( + "INSERT INTO position VALUES (?,?,?,?,?,?,?,?)", + (doc_id, _txt(p, "UniqueID"), _txt(p, "Identifiers/ISIN"), + _txt(p, "Currency"), + float(p.xpath(f"TotalValue/Amount[@ccy='{ccy}']")[0].text), + float(_txt(p, "TotalPercentage")), + kind, qty)) + con.commit() + con.close() + return doc_id + + +def _first(node, path): + r = node.xpath(path) + return r[0].text if r else None + + +def _num(v): + return float(v) if v not in (None, "") else None + + +# ------------------------------------------------------------ generate ------- +def _el(parent, tag, text=None, **attrs): + e = etree.SubElement(parent, tag) + for k, v in attrs.items(): + e.set(k, str(v)) + if text is not None: + e.text = str(text) + return e + + +def generate(db: str, document_id: str, out: str) -> None: + con = sqlite3.connect(db) + con.row_factory = sqlite3.Row + f = con.execute("SELECT * FROM fund WHERE document_id=?", + (document_id,)).fetchone() + if f is None: + raise SystemExit(f"no fund with document_id {document_id!r}") + ccy = f["currency"] + + root = etree.Element("FundsXML4", nsmap={"xsi": XSI}) + root.set(f"{{{XSI}}}noNamespaceSchemaLocation", SCHEMA_URL) + + cd = _el(root, "ControlData") + _el(cd, "UniqueDocumentID", f["document_id"]) + _el(cd, "DocumentGenerated", "2025-10-02T00:00:00") + if f["fxml_version"]: + _el(cd, "Version", f["fxml_version"]) + _el(cd, "ContentDate", f["content_date"]) + ds = _el(cd, "DataSupplier") + _el(ds, "SystemCountry", "AT") + _el(ds, "Short", "EURAM") + _el(ds, "Name", "Erste Asset Management GmbH") + _el(ds, "Type", "Asset Manager") + _el(cd, "DataOperation", "INITIAL") + + fund = _el(_el(root, "Funds"), "Fund") + if f["lei"]: + _el(_el(fund, "Identifiers"), "LEI", f["lei"]) + _el(_el(fund, "Names"), "OfficialName", f["official_name"]) + _el(fund, "Currency", ccy) + _el(fund, "SingleFundFlag", "true") + + fdd = _el(fund, "FundDynamicData") + tav = _el(_el(_el(fdd, "TotalAssetValues"), "TotalAssetValue"), "NavDate", + f["nav_date"]).getparent() + _el(tav, "TotalAssetNature", "OFFICIAL") + _el(_el(tav, "TotalNetAssetValue"), "Amount", + f'{f["total_nav"]:.2f}', ccy=ccy) + + port = _el(_el(fdd, "Portfolios"), "Portfolio") + _el(port, "NavDate", f["nav_date"]) + poss = _el(port, "Positions") + for p in con.execute( + "SELECT * FROM position WHERE document_id=? ORDER BY unique_id", + (document_id,)): + pos = _el(poss, "Position") + _el(pos, "UniqueID", p["unique_id"]) + if p["isin"]: + _el(_el(pos, "Identifiers"), "ISIN", p["isin"]) + if p["currency"]: + _el(pos, "Currency", p["currency"]) + _el(_el(pos, "TotalValue"), "Amount", + f'{p["value_fund_ccy"]:.2f}', ccy=ccy) + _el(pos, "TotalPercentage", f'{p["percentage"]:.2f}') + # Required Position class element. Kinds with a mandatory quantity + # child get it back (Units/Nominal/Shares/Contracts); the rest are + # schema-valid empty. + kind = p["kind"] if p["kind"] in POSITION_KINDS else "Generic" + ke = _el(pos, kind) + if kind in QTY_ELEM and p["kind_qty"] is not None: + _el(ke, QTY_ELEM[kind], f'{p["kind_qty"]:.2f}') + + scs = con.execute("SELECT * FROM share_class WHERE document_id=? " + "ORDER BY isin", (document_id,)).fetchall() + if scs: + sce = _el(_el(fund, "SingleFund"), "ShareClasses") + for sc in scs: + x = _el(sce, "ShareClass") + _el(_el(x, "Identifiers"), "ISIN", sc["isin"]) + if sc["official_name"]: + _el(_el(x, "Names"), "OfficialName", sc["official_name"]) + _el(x, "Currency", sc["currency"]) + if sc["nav_price"] is not None: + pr = _el(_el(x, "Prices"), "Price") + _el(pr, "ActionCode", "C") + _el(pr, "NavDate", f["nav_date"]) + _el(pr, "PriceCurrency", sc["currency"]) + _el(pr, "PriceNature", "OFFICIAL") + _el(pr, "NavPrice", f'{sc["nav_price"]:.2f}') + if sc["nav_fund_ccy"] is not None: + t = _el(_el(x, "TotalAssetValues"), "TotalAssetValue") + _el(t, "NavDate", f["nav_date"]) + _el(t, "TotalAssetNature", "OFFICIAL") + _el(_el(t, "TotalNetAssetValue"), "Amount", + f'{sc["nav_fund_ccy"]:.2f}', ccy=ccy) + if sc["shares_outstanding"] is not None: + _el(t, "SharesOutstanding", + f'{sc["shares_outstanding"]:.0f}') + + assets = con.execute("SELECT * FROM asset WHERE document_id=? " + "ORDER BY unique_id", (document_id,)).fetchall() + if assets: + amd = _el(root, "AssetMasterData") + for a in assets: + ae = _el(amd, "Asset") + _el(ae, "UniqueID", a["unique_id"]) + if a["isin"]: + _el(_el(ae, "Identifiers"), "ISIN", a["isin"]) + _el(ae, "Currency", a["currency"] or ccy) + if a["country"]: + _el(ae, "Country", a["country"]) + _el(ae, "Name", a["name"]) + _el(ae, "AssetType", a["asset_type"]) + con.close() + + etree.ElementTree(root).write(out, xml_declaration=True, + encoding="UTF-8", pretty_print=True) + + +# ---------------------------------------------------------------- cli -------- +def _init(db: str) -> None: + con = sqlite3.connect(db) + con.executescript(DDL.read_text()) + con.commit() + con.close() + + +def main() -> int: + a = sys.argv[1:] + if not a: + print(__doc__, file=sys.stderr) + return 2 + if a[0] == "init": + _init(a[1]) + elif a[0] == "load": + print("loaded document_id:", load(a[1], a[2])) + elif a[0] == "generate": + generate(a[1], a[2], a[3]) + print("wrote", a[3]) + elif a[0] == "roundtrip": + with tempfile.TemporaryDirectory() as t: + db = str(Path(t) / "fundsxml.db") + _init(db) + doc_id = load(db, a[1]) + generate(db, doc_id, a[2]) + print(f"round-trip ok: {a[1]} -> DB -> {a[2]} (doc {doc_id})") + else: + print(f"unknown subcommand {a[0]!r}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/README.md b/README.md index ecb15bb..e5db703 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,8 @@ locations. Items marked _(planned)_ are on the roadmap (see | Schema fetch (proxy-aware) | Bash | [tools/fetch-schema.sh](./tools/fetch-schema.sh) | ✅ | | CI (validate all samples) | GitHub Actions | [.github/workflows/ci.yml](./.github/workflows/) | ✅ | | XQuery analytics (aggregation, top-holdings, look-through) | Saxon CLI/Java, Python, BaseX | [XQuery_Examples/](./XQuery_Examples/) | ✅ | -| XML signature sign/verify | Apache Santuario, .NET, xmlsec1, signxml | `XML_Signature/` | _(planned)_ | -| Database load ↔ generate | Oracle/SQL Server/Postgres (code only) | `Database_Integration/` | _(planned)_ | +| XML signature sign/verify | Apache Santuario (Java), .NET, xmlsec1, signxml | [XML_Signature/](./XML_Signature/) | ✅ | +| Database load ↔ generate | SQLite (verified) + Oracle/SQL Server/Postgres (code) | [Database_Integration/](./Database_Integration/) | ✅ | | Large-file/stream processing | StAX/SAX/lxml iterparse | `Large_File_Processing/` | _(planned)_ | ## Repository Structure diff --git a/XML_Signature/README.md b/XML_Signature/README.md new file mode 100644 index 0000000..ba00ae3 --- /dev/null +++ b/XML_Signature/README.md @@ -0,0 +1,65 @@ +# XML Signature (XML-DSig) + +![status](https://img.shields.io/badge/Java%20(Santuario)-verified-brightgreen) ![profile](https://img.shields.io/badge/RSA--SHA256%20%2F%20exc--C14N%20%2F%20enveloped-blue) + +Sign and verify FundsXML with **enveloped XML Digital Signatures**. All stacks +use the same profile so signed files **cross-verify** between them: + +| Property | Value | +|----------|-------| +| Signature method | RSA-SHA256 | +| Digest | SHA-256 | +| Canonicalization | Exclusive C14N (`xml-exc-c14n#`) | +| Transform | enveloped-signature + exclusive C14N | +| Reference | `URI=""` (whole document) | +| KeyInfo | signer X.509 certificate embedded | +| Placement | `ds:Signature` is the **last child of ``** — exactly where the 4.2.9 schema allows it (`xmldsig-core-schema.xsd` import) | + +A signed file **stays XSD-valid** (verified) and matches the structure of the +committed [`FundsXML_Files/4.2.9/signed/Signed_Fund_Skeleton.xml`](../FundsXML_Files/4.2.9/signed/) +placeholder. + +## Keys + +```bash +XML_Signature/generate-test-key.sh # -> XML_Signature/keys/ (gitignored) +``` +Throwaway self-signed RSA-2048 (`test-signing.p12` alias `fundsxml`, pass +`changeit`; plus PEM key/cert). **Demo only — never commit private keys.** + +## Stacks + +| Stack | Entry point | Status | +|-------|-------------|--------| +| Java — Apache Santuario | [`java/SignFundsXml.java`](java/SignFundsXml.java) / [`java/VerifyFundsXml.java`](java/VerifyFundsXml.java) | ✅ verified (sign, verify, tamper-detect) | +| CLI — `xmlsec1` | [`cli/sign-verify-xmlsec1.sh`](cli/sign-verify-xmlsec1.sh) | reference (needs `xmlsec1`) | +| Python — `signxml` | [`python/sign_verify_signxml.py`](python/sign_verify_signxml.py) | reference (`pip install signxml`) | +| .NET — `SignedXml` | [`dotnet/SignVerify.cs`](dotnet/SignVerify.cs) | reference (needs .NET SDK) | + +## Run (Java / Apache Santuario — verified) + +```bash +tools/fetch-tools.sh +XML_Signature/generate-test-key.sh +CP=.lib/xmlsec-4.0.4.jar:.lib/commons-codec-1.18.0.jar:.lib/slf4j-api-2.0.17.jar:.lib/slf4j-nop-2.0.17.jar +javac -cp "$CP" -d /tmp/sig XML_Signature/java/SignFundsXml.java XML_Signature/java/VerifyFundsXml.java + +# sign +java -cp "$CP:/tmp/sig" SignFundsXml \ + FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml signed.xml \ + XML_Signature/keys/test-signing.p12 changeit fundsxml + +# verify — pin the signer cert (don't trust only the embedded key) +java -cp "$CP:/tmp/sig" VerifyFundsXml signed.xml XML_Signature/keys/test-signing-cert.pem +``` + +`VerifyFundsXml` exits 0 on a valid signature, 1 on tamper/failure (verified: +flipping one digit in a signed file → `INVALID`). Santuario verification runs +with **secure validation** enabled. + +> **Note on `xmlsec1`:** it signs an *existing* `ds:Signature` template, so it +> pairs naturally with the committed signed skeleton; the Java/.NET/Python +> examples instead build and append the `ds:Signature` themselves. + +A real signed file is **not committed** — the signature is bound to the +throwaway key, which is regenerated per run. CI signs → verifies as a roundtrip. diff --git a/XML_Signature/cli/sign-verify-xmlsec1.sh b/XML_Signature/cli/sign-verify-xmlsec1.sh new file mode 100644 index 0000000..cc0a9f8 --- /dev/null +++ b/XML_Signature/cli/sign-verify-xmlsec1.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Enveloped XML-DSig sign / verify on the command line with xmlsec1 +# (the xmlsec C library CLI, package: libxmlsec1 / xmlsec1). +# +# sign-verify-xmlsec1.sh sign +# sign-verify-xmlsec1.sh verify [cert.pem] +# +# Reference implementation (xmlsec1 not installed in the dev environment: +# Debian/Ubuntu: sudo apt-get install xmlsec1 +# macOS: brew install xmlsec1 +# +# Unlike the Java/Python examples, xmlsec1 signs an EXISTING +# template in the document — so it pairs naturally with the committed +# FundsXML_Files/4.2.9/signed/Signed_Fund_Skeleton.xml (which already carries a +# placeholder ds:Signature). Keys: XML_Signature/generate-test-key.sh. +set -euo pipefail + +MODE="${1:?usage: sign-verify-xmlsec1.sh sign|verify ...}" +KEYS="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/keys" + +case "$MODE" in + sign) + IN="${2:?in.xml}"; OUT="${3:?out.xml}" + # The template's ds:Signature/SignedInfo must already exist (it does in the + # signed skeleton). xmlsec1 fills DigestValue + SignatureValue + KeyInfo. + xmlsec1 --sign \ + --privkey-pem "${KEYS}/test-signing-key.pem,${KEYS}/test-signing-cert.pem" \ + --output "$OUT" "$IN" + echo "signed -> $OUT" + ;; + verify) + SIGNED="${2:?signed.xml}" + CERT="${3:-${KEYS}/test-signing-cert.pem}" + # --trusted-pem pins the signer cert (do NOT trust only the embedded key). + xmlsec1 --verify --trusted-pem "$CERT" "$SIGNED" \ + && echo "VALID: signature OK" \ + || { echo "INVALID: signature check failed"; exit 1; } + ;; + *) + echo "unknown mode: $MODE" >&2; exit 2 ;; +esac diff --git a/XML_Signature/dotnet/SignVerify.cs b/XML_Signature/dotnet/SignVerify.cs new file mode 100644 index 0000000..fe42201 --- /dev/null +++ b/XML_Signature/dotnet/SignVerify.cs @@ -0,0 +1,91 @@ +// Enveloped XML-DSig sign / verify in .NET via System.Security.Cryptography.Xml. +// +// dotnet run --project XML_Signature/dotnet -- sign in.xml out.xml +// dotnet run --project XML_Signature/dotnet -- verify signed.xml [cert.pem] +// Exit: 0 ok, 1 invalid, 2 setup error. +// +// Reference implementation — not executed in the dev environment (no .NET SDK). +// Same profile as the Apache Santuario (Java) example: RSA-SHA256, exclusive +// C14N, enveloped, signer cert embedded in KeyInfo, so files cross-verify. +// Keys: XML_Signature/generate-test-key.sh (PKCS#12 test-signing.p12). + +using System; +using System.IO; +using System.Security.Cryptography.X509Certificates; +using System.Security.Cryptography.Xml; +using System.Xml; + +internal static class SignVerify +{ + private static int Main(string[] args) + { + if (args.Length < 2) { Console.Error.WriteLine("usage: sign|verify ..."); return 2; } + string keysDir = Path.Combine(AppContext.BaseDirectory, + "..", "..", "..", "..", "keys"); + + var doc = new XmlDocument { PreserveWhitespace = true }; + // XXE-hardened load. + using (var r = XmlReader.Create(args[1], + new XmlReaderSettings { DtdProcessing = DtdProcessing.Prohibit, + XmlResolver = null })) + doc.Load(r); + + if (args[0] == "sign") + { + var cert = new X509Certificate2( + Path.Combine(keysDir, "test-signing.p12"), "changeit", + X509KeyStorageFlags.Exportable); + var rsa = cert.GetRSAPrivateKey(); + + var signedXml = new SignedXml(doc) { SigningKey = rsa }; + signedXml.SignedInfo.CanonicalizationMethod = + SignedXml.XmlDsigExcC14NTransformUrl; + signedXml.SignedInfo.SignatureMethod = + "http://www.w3.org/2001/04/xmldsig-more#rsa-sha256"; + + var reference = new Reference(""); + reference.DigestMethod = "http://www.w3.org/2001/04/xmlenc#sha256"; + reference.AddTransform(new XmlDsigEnvelopedSignatureTransform()); + reference.AddTransform(new XmlDsigExcC14NTransform()); + signedXml.AddReference(reference); + + var ki = new KeyInfo(); + ki.AddClause(new KeyInfoX509Data(cert)); + signedXml.KeyInfo = ki; + + signedXml.ComputeSignature(); + // ds:Signature must be the LAST child of . + doc.DocumentElement!.AppendChild( + doc.ImportNode(signedXml.GetXml(), true)); + doc.Save(args[2]); + Console.WriteLine($"signed -> {args[2]}"); + return 0; + } + + if (args[0] == "verify") + { + var signedXml = new SignedXml(doc); + var sig = (XmlElement)doc.GetElementsByTagName( + "Signature", SignedXml.XmlDsigNamespaceUrl)[0]!; + signedXml.LoadXml(sig); + + bool ok; + if (args.Length >= 3) + { + var pinned = new X509Certificate2(args[2]); + ok = signedXml.CheckSignature(pinned, true); + Console.WriteLine($"pinned cert: {pinned.Subject}"); + } + else + { + ok = signedXml.CheckSignature(); // trusts embedded key (demo) + } + Console.WriteLine(ok ? "VALID: signature OK" + : "INVALID: signature check failed"); + return ok ? 0 : 1; + } + + Console.Error.WriteLine($"unknown mode {args[0]}"); + return 2; + } +} diff --git a/XML_Signature/dotnet/SignVerify.csproj b/XML_Signature/dotnet/SignVerify.csproj new file mode 100644 index 0000000..f23f5af --- /dev/null +++ b/XML_Signature/dotnet/SignVerify.csproj @@ -0,0 +1,15 @@ + + + + Exe + net8.0 + enable + SignVerify + FundsXml.XmlSignature + + + + + diff --git a/XML_Signature/generate-test-key.sh b/XML_Signature/generate-test-key.sh new file mode 100755 index 0000000..77dd220 --- /dev/null +++ b/XML_Signature/generate-test-key.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# generate-test-key.sh — create a THROWAWAY self-signed RSA key + cert for the +# XML-signature examples. +# +# Output (into XML_Signature/keys/, gitignored — never commit private keys): +# test-signing.p12 PKCS#12 keystore (alias: fundsxml, pass: changeit) +# test-signing-cert.pem public certificate (for verification / xmlsec1) +# test-signing-key.pem private key in PEM (for xmlsec1 / signxml) +# +# FOR DEMO USE ONLY — 2048-bit, 10-year self-signed, hard-coded password. +set -euo pipefail + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/keys" +mkdir -p "$DIR" +PASS="changeit" + +openssl req -x509 -newkey rsa:2048 -sha256 -days 3650 -nodes \ + -keyout "$DIR/test-signing-key.pem" \ + -out "$DIR/test-signing-cert.pem" \ + -subj "/C=AT/O=FundsXML Examples/OU=Demo/CN=fundsxml-test-signer" 2>/dev/null + +openssl pkcs12 -export \ + -inkey "$DIR/test-signing-key.pem" \ + -in "$DIR/test-signing-cert.pem" \ + -name fundsxml \ + -out "$DIR/test-signing.p12" \ + -passout "pass:${PASS}" + +echo "wrote: $DIR/test-signing.p12 (alias=fundsxml pass=${PASS})" +echo " $DIR/test-signing-cert.pem $DIR/test-signing-key.pem" diff --git a/XML_Signature/java/SignFundsXml.java b/XML_Signature/java/SignFundsXml.java new file mode 100644 index 0000000..4196cf3 --- /dev/null +++ b/XML_Signature/java/SignFundsXml.java @@ -0,0 +1,90 @@ +// SignFundsXml — enveloped XML-DSig signing with Apache Santuario (xmlsec 4.x). +// +// tools/fetch-tools.sh +// XML_Signature/generate-test-key.sh +// CP=.lib/xmlsec-4.0.4.jar:.lib/commons-codec-1.18.0.jar:\ +// .lib/slf4j-api-2.0.17.jar:.lib/slf4j-nop-2.0.17.jar +// javac -cp "$CP" -d /tmp/sig XML_Signature/java/SignFundsXml.java +// java -cp "$CP:/tmp/sig" SignFundsXml \ +// XML_Signature/keys/test-signing.p12 changeit fundsxml +// +// Produces an enveloped signature whose ds:Signature is appended as the last +// child of — exactly where the FundsXML 4.2.9 schema allows it +// (xmldsig-core-schema.xsd import). The signer certificate is embedded in +// KeyInfo/X509Data so the signed file is self-verifiable. +// +// Security: DocumentBuilderFactory is namespace-aware and XXE-hardened. + +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.security.Key; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import org.apache.xml.security.Init; +import org.apache.xml.security.algorithms.MessageDigestAlgorithm; +import org.apache.xml.security.c14n.Canonicalizer; +import org.apache.xml.security.signature.XMLSignature; +import org.apache.xml.security.transforms.Transforms; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +public class SignFundsXml { + public static void main(String[] args) throws Exception { + if (args.length != 5) { + System.err.println("usage: SignFundsXml " + + " "); + System.exit(2); + } + String in = args[0], out = args[1], ks = args[2], + pass = args[3], alias = args[4]; + + Init.init(); + + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + dbf.setNamespaceAware(true); + dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + dbf.setFeature("http://xml.org/sax/features/external-general-entities", false); + dbf.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + Document doc; + try (FileInputStream fis = new FileInputStream(in)) { + doc = dbf.newDocumentBuilder().parse(fis); + } + + KeyStore keyStore = KeyStore.getInstance("PKCS12"); + try (FileInputStream kfis = new FileInputStream(ks)) { + keyStore.load(kfis, pass.toCharArray()); + } + Key key = keyStore.getKey(alias, pass.toCharArray()); + X509Certificate cert = (X509Certificate) keyStore.getCertificate(alias); + + Element root = doc.getDocumentElement(); + XMLSignature sig = new XMLSignature(doc, "", + XMLSignature.ALGO_ID_SIGNATURE_RSA_SHA256, + Canonicalizer.ALGO_ID_C14N_EXCL_OMIT_COMMENTS); + + // ds:Signature is the LAST allowed child of . + root.appendChild(sig.getElement()); + + Transforms tr = new Transforms(doc); + tr.addTransform(Transforms.TRANSFORM_ENVELOPED_SIGNATURE); + tr.addTransform(Canonicalizer.ALGO_ID_C14N_EXCL_OMIT_COMMENTS); + sig.addDocument("", tr, MessageDigestAlgorithm.ALGO_ID_DIGEST_SHA256); + + sig.addKeyInfo(cert); + sig.addKeyInfo(cert.getPublicKey()); + sig.sign((PrivateKey) key); + + Transformer t = TransformerFactory.newInstance().newTransformer(); + try (FileOutputStream fos = new FileOutputStream(out)) { + t.transform(new DOMSource(doc), new StreamResult(fos)); + } + System.out.println("signed -> " + out + + " (RSA-SHA256, exclusive C14N, enveloped)"); + } +} diff --git a/XML_Signature/java/VerifyFundsXml.java b/XML_Signature/java/VerifyFundsXml.java new file mode 100644 index 0000000..0141a83 --- /dev/null +++ b/XML_Signature/java/VerifyFundsXml.java @@ -0,0 +1,75 @@ +// VerifyFundsXml — verify an enveloped XML-DSig signature with Apache Santuario. +// +// java -cp "$CP:/tmp/sig" VerifyFundsXml [cert.pem] +// +// With no cert argument the certificate embedded in KeyInfo/X509Data is used +// (self-verifiable file). Pass a PEM cert to pin verification to a known key +// (recommended in production — never trust the key shipped inside the document +// alone). Exit: 0 = signature valid, 1 = invalid, 2 = setup error. +// +// Security: namespace-aware, XXE-hardened parser; "secure validation" mode on. + +import java.io.FileInputStream; +import java.security.PublicKey; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import javax.xml.parsers.DocumentBuilderFactory; +import org.apache.xml.security.Init; +import org.apache.xml.security.signature.XMLSignature; +import org.apache.xml.security.utils.Constants; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +public class VerifyFundsXml { + public static void main(String[] args) throws Exception { + if (args.length < 1) { + System.err.println("usage: VerifyFundsXml [cert.pem]"); + System.exit(2); + } + Init.init(); + + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + dbf.setNamespaceAware(true); + dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + dbf.setFeature("http://xml.org/sax/features/external-general-entities", false); + dbf.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + Document doc; + try (FileInputStream fis = new FileInputStream(args[0])) { + doc = dbf.newDocumentBuilder().parse(fis); + } + + Element sigEl = (Element) doc.getElementsByTagNameNS( + Constants.SignatureSpecNS, "Signature").item(0); + if (sigEl == null) { + System.err.println("INVALID: no ds:Signature element found"); + System.exit(1); + } + + // xmlsec 4.x: secure validation is a constructor argument (blocks + // RetrievalMethod loops, dangerous transforms, weak algorithms, etc.). + XMLSignature signature = new XMLSignature(sigEl, "", true); + + boolean ok; + if (args.length >= 2) { + try (FileInputStream cf = new FileInputStream(args[1])) { + X509Certificate pinned = (X509Certificate) CertificateFactory + .getInstance("X.509").generateCertificate(cf); + ok = signature.checkSignatureValue(pinned.getPublicKey()); + System.out.println("verifying against pinned cert: " + + pinned.getSubjectX500Principal()); + } + } else { + X509Certificate embedded = + signature.getKeyInfo().getX509Certificate(); + PublicKey pk = embedded != null ? embedded.getPublicKey() + : signature.getKeyInfo().getPublicKey(); + ok = signature.checkSignatureValue(pk); + System.out.println("verifying against KeyInfo-embedded key" + + (embedded != null ? " (cert: " + + embedded.getSubjectX500Principal() + ")" : "")); + } + + System.out.println(ok ? "VALID: signature OK" : "INVALID: signature check failed"); + System.exit(ok ? 0 : 1); + } +} diff --git a/XML_Signature/python/sign_verify_signxml.py b/XML_Signature/python/sign_verify_signxml.py new file mode 100644 index 0000000..7c064f4 --- /dev/null +++ b/XML_Signature/python/sign_verify_signxml.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Enveloped XML-DSig sign / verify in Python via `signxml`. + +Usage: + python sign_verify_signxml.py sign + python sign_verify_signxml.py verify [cert.pem] + +Exit: 0 ok, 1 invalid signature, 2 setup error. + +Reference implementation (signxml not installed in the dev environment): + pip install signxml +Keys come from XML_Signature/generate-test-key.sh (key/cert PEM in +XML_Signature/keys/). RSA-SHA256, exclusive C14N, enveloped — same profile as +the Apache Santuario (Java) example, so files cross-verify between stacks. +""" +import sys +from pathlib import Path + +KEYS = Path(__file__).resolve().parents[1] / "keys" + + +def main() -> int: + if len(sys.argv) < 3: + print(__doc__, file=sys.stderr) + return 2 + mode = sys.argv[1] + + try: + from lxml import etree + from signxml import XMLSigner, XMLVerifier, methods + except ImportError: + print("signxml not installed. Run: pip install signxml", + file=sys.stderr) + return 2 + + if mode == "sign": + src, out = sys.argv[2], sys.argv[3] + data = etree.parse(src).getroot() + key = (KEYS / "test-signing-key.pem").read_bytes() + cert = (KEYS / "test-signing-cert.pem").read_bytes() + signed = XMLSigner( + method=methods.enveloped, + signature_algorithm="rsa-sha256", + digest_algorithm="sha256", + c14n_algorithm="http://www.w3.org/2001/10/xml-exc-c14n#", + ).sign(data, key=key, cert=cert) + Path(out).write_bytes(etree.tostring(signed)) + print(f"signed -> {out}") + return 0 + + if mode == "verify": + signed = etree.parse(sys.argv[2]).getroot() + try: + if len(sys.argv) >= 4: + XMLVerifier().verify( + signed, x509_cert=Path(sys.argv[3]).read_text()) + else: + # Trust the embedded cert ONLY for the demo; in production pin + # x509_cert / ca_pem_file to a known signer. + XMLVerifier().verify(signed) + print("VALID: signature OK") + return 0 + except Exception as e: # signxml raises on any failure + print(f"INVALID: {e}") + return 1 + + print(f"unknown mode {mode!r}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/fetch-tools.sh b/tools/fetch-tools.sh index 9eaa120..2e589b9 100755 --- a/tools/fetch-tools.sh +++ b/tools/fetch-tools.sh @@ -44,5 +44,14 @@ fetch "${M2}/commons-cli/commons-cli/1.5.0/commons-cli-1.5.0.jar" "${LIB}/common fetch "${M2}/org/slf4j/slf4j-api/1.7.32/slf4j-api-1.7.32.jar" "${LIB}/slf4j-api-1.7.32.jar" fetch "${M2}/org/slf4j/slf4j-nop/1.7.32/slf4j-nop-1.7.32.jar" "${LIB}/slf4j-nop-1.7.32.jar" +# --- Apache Santuario (XML Signature, Phase 3) ------------------------------ +# xmlsec 4.0.4 runtime needs commons-codec; slf4j-api for logging (+ a no-op +# binding to silence "no SLF4J providers" on the console). +fetch "${M2}/org/apache/santuario/xmlsec/4.0.4/xmlsec-4.0.4.jar" "${LIB}/xmlsec-4.0.4.jar" +fetch "${M2}/commons-codec/commons-codec/1.18.0/commons-codec-1.18.0.jar" "${LIB}/commons-codec-1.18.0.jar" +fetch "${M2}/org/slf4j/slf4j-api/2.0.17/slf4j-api-2.0.17.jar" "${LIB}/slf4j-api-2.0.17.jar" +fetch "${M2}/org/slf4j/slf4j-nop/2.0.17/slf4j-nop-2.0.17.jar" "${LIB}/slf4j-nop-2.0.17.jar" + echo "SAXON_CP=${LIB}/Saxon-HE-12.5.jar:${LIB}/xmlresolver-5.2.2.jar:${LIB}/xmlresolver-5.2.2-data.jar" echo "SCHXSLT_CP=${LIB}/schxslt-cli-1.10.1.jar:${LIB}/commons-cli-1.5.0.jar:${LIB}/slf4j-api-1.7.32.jar:${LIB}/slf4j-nop-1.7.32.jar" +echo "SANTUARIO_CP=${LIB}/xmlsec-4.0.4.jar:${LIB}/commons-codec-1.18.0.jar:${LIB}/slf4j-api-2.0.17.jar:${LIB}/slf4j-nop-2.0.17.jar"