Phase 3 + 4: XML signature (re-landed) + database integration

.github/workflows/ci.yml

@@ -100,6 +100,45 @@ jobs:
           done
           test "$(xmllint --xpath 'count(//Holding)' out_top.xml)" = "5"
 
+      - name: XML signature - Apache Santuario sign/verify/tamper roundtrip
+        run: |
+          set -e
+          XML_Signature/generate-test-key.sh
+          CP=.lib/xmlsec-4.0.4.jar:.lib/commons-codec-1.18.0.jar:.lib/slf4j-api-2.0.17.jar:.lib/slf4j-nop-2.0.17.jar
+          javac -cp "$CP" -d /tmp/sig \
+            XML_Signature/java/SignFundsXml.java XML_Signature/java/VerifyFundsXml.java
+          SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml
+          java -cp "$CP:/tmp/sig" SignFundsXml "$SRC" signed.xml \
+            XML_Signature/keys/test-signing.p12 changeit fundsxml
+          java -cp "$CP:/tmp/sig" VerifyFundsXml signed.xml \
+            XML_Signature/keys/test-signing-cert.pem
+          xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd signed.xml
+          sed 's/<TotalPercentage>8.33</<TotalPercentage>9.33</' signed.xml > tampered.xml
+          if java -cp "$CP:/tmp/sig" VerifyFundsXml tampered.xml \
+               XML_Signature/keys/test-signing-cert.pem; then
+            echo "::error::tampered file unexpectedly verified"; exit 1
+          fi
+          echo "tamper correctly detected"
+
+      - name: DB integration - FundsXML -> SQLite -> FundsXML round-trip
+        run: |
+          set -e
+          python3 -m pip install --quiet lxml
+          SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml
+          python3 Database_Integration/python/fundsxml_db.py roundtrip "$SRC" regen.xml
+          xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd regen.xml
+          python3 - "$SRC" regen.xml <<'PY'
+          import re, sys
+          o, r = (open(p).read() for p in sys.argv[1:3])
+          nav = lambda x: re.search(r'<TotalNetAssetValue>\s*<Amount ccy="EUR">([0-9.]+)', x).group(1)
+          npos = lambda x: len(re.findall(r'<Position>', x))
+          spct = lambda x: round(sum(float(v) for v in re.findall(r'<TotalPercentage>([0-9.]+)', x)), 2)
+          assert abs(float(nav(o)) - float(nav(r))) < 0.01, (nav(o), nav(r))
+          assert npos(o) == npos(r), (npos(o), npos(r))
+          assert abs(spct(o) - spct(r)) < 0.01, (spct(o), spct(r))
+          print("round-trip figures preserved:", nav(r), npos(r), spct(r))
+          PY
+
       - name: Regression - legacy XSLT 1.0 report still runs
         run: |
           xsltproc XSLT_DataQuality_Checks/Enhanced_Check/FundsXML_CompleteDQReport_HTML.xsl \

.gitignore

@@ -8,6 +8,10 @@ CLAUDE.md
 # dependencies from Maven Central, not repo source.
 .lib/
 
+# Throwaway signing keys (XML_Signature/generate-test-key.sh) — never commit
+# private keys, even demo ones.
+XML_Signature/keys/
+
 # Generated reports / outputs
 *.report.html
 report.html
@@ -18,4 +22,7 @@ out_*.html
 out_*.fo
 out_*.csv
 out_*.xml
+regen.xml
+regenerated.xml
+*.db
 *.svrl

Database_Integration/README.md

@@ -0,0 +1,48 @@
+# Database Integration
+
+![python](https://img.shields.io/badge/SQLite%20round--trip-verified-brightgreen) ![sql](https://img.shields.io/badge/Oracle%2FSQLServer%2FPostgres-code--reference-blue)
+
+FundsXML ⇄ relational database, both directions. Per project scope **no
+database is provisioned** (no Docker): the Oracle / SQL Server / PostgreSQL
+files are **code references**, and the **SQLite Python implementation is the
+runnable, verified reference** (stdlib + lxml only) that exercises the full
+round-trip and produces XSD-valid FundsXML.
+
+## Layout
+
+| Path | What |
+|------|------|
+| [`ddl/schema.sql`](ddl/schema.sql) | Shared relational model (fund, share_class, asset, position) |
+| [`load_from_fundsxml/`](load_from_fundsxml/) | FundsXML → DB: `postgres.sql` (XMLTABLE), `oracle.sql` (XMLType + XMLTABLE), `sqlserver.sql` (`.nodes()`/`.value()`) |
+| [`generate_fundsxml/`](generate_fundsxml/) | DB → FundsXML: SQL/XML publishing (`xmlelement`/`xmlagg`, `XMLElement`/`XMLAgg`, `FOR XML PATH`) |
+| [`python/fundsxml_db.py`](python/fundsxml_db.py) | **Runnable** SQLite reference: `init` / `load` / `generate` / `roundtrip` |
+
+The relational model is keyed by `ControlData/UniqueDocumentID` so multiple
+documents coexist. FundsXML 4.x has no XML namespace — all XPath/queries use
+bare element names.
+
+## Runnable round-trip (verified)
+
+```bash
+python3 Database_Integration/python/fundsxml_db.py roundtrip \
+  FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml regenerated.xml
+
+tools/fetch-schema.sh 4.2.9
+xmllint --noout --schema .schema-cache/4.2.9/FundsXML.xsd regenerated.xml
+```
+
+Verified for the 4.2.9 positions, 4.1.0 and 4.0.0 positions, and 4.2.9
+transactions samples: each loads into SQLite and regenerates **XSD-valid**
+FundsXML with NAV, position count and percentage sum preserved.
+
+### Round-trip fidelity
+
+The model captures fund header, share classes (incl. `TotalAssetValues` /
+`SharesOutstanding`), positions (value, %, class + quantity child) and asset
+master data. It is intentionally **lossy** for everything outside that core
+(issuer details, derivative terms, transactions, regulatory blocks) — the
+regenerated file is a faithful *positions* document, normalized to the 4.2.9
+schema (a 4.0.0 input comes back without `ControlData/Version`, which is valid).
+
+The three enterprise-DB SQL files mirror the same shred/publish logic; only the
+SQLite path is executed here. CI runs the round-trip on every push.

Database_Integration/ddl/schema.sql

@@ -0,0 +1,65 @@
+-- Shared relational model for FundsXML positions data.
+--
+-- Portable ANSI-ish DDL (works on SQLite/Postgres; see notes for Oracle and
+-- SQL Server type tweaks). One row set per source document, keyed by the
+-- ControlData/UniqueDocumentID so multiple files can coexist.
+--
+-- Faithful enough to round-trip the canonical positions sample back to
+-- XSD-valid FundsXML (fund + share classes + positions + asset master data).
+
+CREATE TABLE fund (
+    document_id     VARCHAR(128) NOT NULL,
+    lei             VARCHAR(20),
+    official_name   VARCHAR(256) NOT NULL,
+    currency        CHAR(3)      NOT NULL,
+    content_date    DATE,
+    nav_date        DATE,
+    total_nav       DECIMAL(20,2) NOT NULL,
+    fxml_version    VARCHAR(16),                -- absent for 4.0.0
+    PRIMARY KEY (document_id)
+);
+
+CREATE TABLE share_class (
+    document_id     VARCHAR(128) NOT NULL,
+    isin            CHAR(12)     NOT NULL,
+    official_name   VARCHAR(256),
+    currency        CHAR(3)      NOT NULL,
+    nav_price       DECIMAL(20,6),
+    nav_fund_ccy    DECIMAL(20,2),              -- NAV in fund currency
+    shares_outstanding DECIMAL(28,6),
+    PRIMARY KEY (document_id, isin),
+    FOREIGN KEY (document_id) REFERENCES fund (document_id)
+);
+
+CREATE TABLE asset (
+    document_id     VARCHAR(128) NOT NULL,
+    unique_id       VARCHAR(256) NOT NULL,
+    isin            CHAR(12),
+    name            VARCHAR(256),
+    asset_type      CHAR(2),                    -- EQ, BO, SC, ...
+    currency        CHAR(3),
+    country         CHAR(2),
+    PRIMARY KEY (document_id, unique_id),
+    FOREIGN KEY (document_id) REFERENCES fund (document_id)
+);
+
+CREATE TABLE position (
+    document_id     VARCHAR(128) NOT NULL,
+    unique_id       VARCHAR(256) NOT NULL,      -- joins to asset.unique_id
+    isin            CHAR(12),
+    currency        CHAR(3),
+    value_fund_ccy  DECIMAL(20,2) NOT NULL,
+    percentage      DECIMAL(9,4)  NOT NULL,
+    kind            VARCHAR(16),                -- Position class element:
+                                                -- Equity/Bond/ShareClass/...
+    kind_qty        DECIMAL(28,6),              -- its quantity child
+                                                -- (Units/Nominal/Shares/Contracts)
+    PRIMARY KEY (document_id, unique_id),
+    FOREIGN KEY (document_id) REFERENCES fund (document_id)
+);
+
+-- Dialect notes
+--   Oracle      : VARCHAR(n) -> VARCHAR2(n); DECIMAL -> NUMBER; CHAR ok.
+--                 An XMLType staging column is shown in load_from_fundsxml.
+--   SQL Server  : VARCHAR ok; DECIMAL ok; use an XML column for staging.
+--   Postgres    : as-is; a native `xml` staging column is used for xmltable.

Database_Integration/generate_fundsxml/oracle.sql

@@ -0,0 +1,60 @@
+-- Oracle -> FundsXML (code reference; no DB is provisioned in this repo).
+--
+-- SQL/XML publishing: XMLElement / XMLAgg / XMLAttributes. Same FundsXML 4.2.9
+-- positions shape as the Python reference. Bind :doc = fund.document_id.
+
+SELECT XMLElement("FundsXML4",
+         XMLAttributes(
+           'http://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd'
+             AS "xsi:noNamespaceSchemaLocation"),
+         XMLElement("ControlData",
+           XMLElement("UniqueDocumentID", f.document_id),
+           XMLElement("DocumentGenerated", '2025-10-02T00:00:00'),
+           CASE WHEN f.fxml_version IS NOT NULL
+                THEN XMLElement("Version", f.fxml_version) END,
+           XMLElement("ContentDate", TO_CHAR(f.content_date,'YYYY-MM-DD')),
+           XMLElement("DataSupplier",
+             XMLElement("SystemCountry",'AT'),
+             XMLElement("Short",'EURAM'),
+             XMLElement("Name",'Erste Asset Management GmbH'),
+             XMLElement("Type",'Asset Manager')),
+           XMLElement("DataOperation",'INITIAL')),
+         XMLElement("Funds",
+           XMLElement("Fund",
+             XMLElement("Identifiers", XMLElement("LEI", f.lei)),
+             XMLElement("Names", XMLElement("OfficialName", f.official_name)),
+             XMLElement("Currency", f.currency),
+             XMLElement("SingleFundFlag",'true'),
+             XMLElement("FundDynamicData",
+               XMLElement("TotalAssetValues",
+                 XMLElement("TotalAssetValue",
+                   XMLElement("NavDate", TO_CHAR(f.nav_date,'YYYY-MM-DD')),
+                   XMLElement("TotalAssetNature",'OFFICIAL'),
+                   XMLElement("TotalNetAssetValue",
+                     XMLElement("Amount",
+                       XMLAttributes(f.currency AS "ccy"),
+                       TO_CHAR(f.total_nav,'FM999999999990.00'))))),
+               XMLElement("Portfolios",
+                 XMLElement("Portfolio",
+                   XMLElement("NavDate", TO_CHAR(f.nav_date,'YYYY-MM-DD')),
+                   XMLElement("Positions",
+                     (SELECT XMLAgg(
+                        XMLElement("Position",
+                          XMLElement("UniqueID", p.unique_id),
+                          XMLElement("Currency", p.currency),
+                          XMLElement("TotalValue",
+                            XMLElement("Amount",
+                              XMLAttributes(f.currency AS "ccy"),
+                              TO_CHAR(p.value_fund_ccy,'FM999999999990.00'))),
+                          XMLElement("TotalPercentage",
+                            TO_CHAR(p.percentage,'FM990.00')),
+                          XMLElement(EVALNAME p.kind))
+                        ORDER BY p.unique_id)
+                      FROM position p
+                      WHERE p.document_id = f.document_id)))))))
+       ).getClobVal()
+FROM fund f
+WHERE f.document_id = :doc;
+
+-- As with the Postgres example, add the quantity child per kind
+-- (position.kind_qty) to keep Equity/Bond/ShareClass/... schema-valid.

Database_Integration/generate_fundsxml/postgres.sql

@@ -0,0 +1,64 @@
+-- PostgreSQL -> FundsXML (code reference; no DB is provisioned in this repo).
+--
+-- SQL/XML publishing with xmlelement/xmlagg/xmlattributes. Produces the same
+-- FundsXML 4.2.9 positions shape the Python reference emits (XSD-valid).
+-- Parameter :doc = fund.document_id.
+
+SELECT xmlelement(name "FundsXML4",
+         xmlattributes('http://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd'
+                       AS "xsi:noNamespaceSchemaLocation"),
+         xmlelement(name "ControlData",
+           xmlelement(name "UniqueDocumentID", f.document_id),
+           xmlelement(name "DocumentGenerated", '2025-10-02T00:00:00'),
+           CASE WHEN f.fxml_version IS NOT NULL
+                THEN xmlelement(name "Version", f.fxml_version) END,
+           xmlelement(name "ContentDate", f.content_date),
+           xmlelement(name "DataSupplier",
+             xmlelement(name "SystemCountry",'AT'),
+             xmlelement(name "Short",'EURAM'),
+             xmlelement(name "Name",'Erste Asset Management GmbH'),
+             xmlelement(name "Type",'Asset Manager')),
+           xmlelement(name "DataOperation",'INITIAL')),
+         xmlelement(name "Funds",
+           xmlelement(name "Fund",
+             xmlelement(name "Identifiers",
+               xmlelement(name "LEI", f.lei)),
+             xmlelement(name "Names",
+               xmlelement(name "OfficialName", f.official_name)),
+             xmlelement(name "Currency", f.currency),
+             xmlelement(name "SingleFundFlag",'true'),
+             xmlelement(name "FundDynamicData",
+               xmlelement(name "TotalAssetValues",
+                 xmlelement(name "TotalAssetValue",
+                   xmlelement(name "NavDate", f.nav_date),
+                   xmlelement(name "TotalAssetNature",'OFFICIAL'),
+                   xmlelement(name "TotalNetAssetValue",
+                     xmlelement(name "Amount",
+                       xmlattributes(f.currency AS "ccy"),
+                       to_char(f.total_nav,'FM999999999990.00'))))),
+               xmlelement(name "Portfolios",
+                 xmlelement(name "Portfolio",
+                   xmlelement(name "NavDate", f.nav_date),
+                   xmlelement(name "Positions",
+                     (SELECT xmlagg(
+                        xmlelement(name "Position",
+                          xmlelement(name "UniqueID", p.unique_id),
+                          CASE WHEN p.isin IS NOT NULL THEN
+                            xmlelement(name "Identifiers",
+                              xmlelement(name "ISIN", p.isin)) END,
+                          xmlelement(name "Currency", p.currency),
+                          xmlelement(name "TotalValue",
+                            xmlelement(name "Amount",
+                              xmlattributes(f.currency AS "ccy"),
+                              to_char(p.value_fund_ccy,'FM999999999990.00'))),
+                          xmlelement(name "TotalPercentage",
+                            to_char(p.percentage,'FM990.00')),
+                          xmlelement(name p.kind)) ORDER BY p.unique_id)
+                      FROM position p WHERE p.document_id = f.document_id))))))
+       )
+FROM fund f
+WHERE f.document_id = :doc;
+
+-- For brevity the position class element is emitted empty; add the quantity
+-- child (Units/Nominal/Shares/Contracts from position.kind_qty) exactly as the
+-- Python reference does to keep Equity/Bond/ShareClass/... schema-valid.

Database_Integration/generate_fundsxml/sqlserver.sql

@@ -0,0 +1,51 @@
+-- SQL Server -> FundsXML (code reference; no DB is provisioned in this repo).
+--
+-- FOR XML PATH with nested subqueries. Same FundsXML 4.2.9 positions shape as
+-- the Python reference. @doc = fund.document_id.
+
+DECLARE @doc varchar(128) = 'FUNDSXML_FILE_1';
+
+SELECT
+  'http://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd'
+    AS [@xsi:noNamespaceSchemaLocation],
+  f.document_id              AS [ControlData/UniqueDocumentID],
+  '2025-10-02T00:00:00'      AS [ControlData/DocumentGenerated],
+  f.fxml_version             AS [ControlData/Version],
+  CONVERT(varchar(10), f.content_date, 23) AS [ControlData/ContentDate],
+  'AT'    AS [ControlData/DataSupplier/SystemCountry],
+  'EURAM' AS [ControlData/DataSupplier/Short],
+  'Erste Asset Management GmbH' AS [ControlData/DataSupplier/Name],
+  'Asset Manager' AS [ControlData/DataSupplier/Type],
+  'INITIAL' AS [ControlData/DataOperation],
+  f.lei           AS [Funds/Fund/Identifiers/LEI],
+  f.official_name AS [Funds/Fund/Names/OfficialName],
+  f.currency      AS [Funds/Fund/Currency],
+  'true'          AS [Funds/Fund/SingleFundFlag],
+  CONVERT(varchar(10), f.nav_date, 23)
+    AS [Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/NavDate],
+  'OFFICIAL'
+    AS [Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/TotalAssetNature],
+  f.currency
+    AS [Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount/@ccy],
+  CONVERT(varchar(32), f.total_nav)
+    AS [Funds/Fund/FundDynamicData/TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount],
+  (SELECT
+       p.unique_id AS [UniqueID],
+       p.currency  AS [Currency],
+       f.currency  AS [TotalValue/Amount/@ccy],
+       CONVERT(varchar(32), p.value_fund_ccy) AS [TotalValue/Amount],
+       CONVERT(varchar(16), p.percentage)     AS [TotalPercentage]
+       -- plus the kind element (+ quantity child) — see note below
+   FROM position p
+   WHERE p.document_id = f.document_id
+   ORDER BY p.unique_id
+   FOR XML PATH('Position'), TYPE)
+     AS [Funds/Fund/FundDynamicData/Portfolios/Portfolio/Positions]
+FROM fund f
+WHERE f.document_id = @doc
+FOR XML PATH('FundsXML4');
+
+-- The position class element (Equity/Bond/.../with its Units/Nominal/Shares/
+-- Contracts child from position.kind_qty) is added with a correlated
+-- FOR XML PATH subquery per row — emitted dynamically because the element name
+-- itself varies; see the Python reference for the exact mapping.