From 36b6931ad60cae68f504cb5e0b24ade5845db3ad Mon Sep 17 00:00:00 2001 From: Karl Kauc Date: Fri, 15 May 2026 22:00:19 +0200 Subject: [PATCH] Database_Integration: split each language into separate import + export Per request: import and export are now separate standalone programs (not one combined file), so a developer lifts exactly the direction they need. - python/ import_fundsxml.py + export_fundsxml.py (was fundsxml_db.py) - java/ ImportFundsXml.java + ExportFundsXml.java (was FundsXmlDb.java) - javascript/ import_fundsxml.mjs + export_fundsxml.mjs (was fundsxml_db.mjs) - csharp/import/ + csharp/export/ (separate .csproj each; was one project) Each is self-contained (own copy of the small kind/qty maps) and heavily commented. Import creates the schema in a fresh SQLite DB then loads; export reads it back. CLI: ; . README + CI updated to run import then export then xml_equiv + XSD for all four languages on the multi-fund fixture (cross-language identical) plus the single-fund sample. Verified locally end-to-end: every language's export is xml_equiv-equal to the input and to each other, and XSD-valid. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 32 +- Database_Integration/README.md | 53 +- .../ExportFundsXml.cs} | 276 ++-------- .../ExportFundsXml.csproj} | 10 +- .../csharp/import/ImportFundsXml.cs | 185 +++++++ .../csharp/import/ImportFundsXml.csproj | 17 + Database_Integration/java/ExportFundsXml.java | 248 +++++++++ Database_Integration/java/FundsXmlDb.java | 493 ------------------ Database_Integration/java/ImportFundsXml.java | 242 +++++++++ .../javascript/export_fundsxml.mjs | 181 +++++++ .../javascript/fundsxml_db.mjs | 317 ----------- .../javascript/import_fundsxml.mjs | 139 +++++ .../python/export_fundsxml.py | 188 +++++++ Database_Integration/python/fundsxml_db.py | 362 ------------- .../python/import_fundsxml.py | 151 ++++++ 15 files changed, 1447 insertions(+), 1447 deletions(-) rename Database_Integration/csharp/{FundsXmlDb.cs => export/ExportFundsXml.cs} (55%) rename Database_Integration/csharp/{FundsXmlDb.csproj => export/ExportFundsXml.csproj} (50%) create mode 100644 Database_Integration/csharp/import/ImportFundsXml.cs create mode 100644 Database_Integration/csharp/import/ImportFundsXml.csproj create mode 100644 Database_Integration/java/ExportFundsXml.java delete mode 100644 Database_Integration/java/FundsXmlDb.java create mode 100644 Database_Integration/java/ImportFundsXml.java create mode 100644 Database_Integration/javascript/export_fundsxml.mjs delete mode 100644 Database_Integration/javascript/fundsxml_db.mjs create mode 100644 Database_Integration/javascript/import_fundsxml.mjs create mode 100644 Database_Integration/python/export_fundsxml.py delete mode 100644 Database_Integration/python/fundsxml_db.py create mode 100644 Database_Integration/python/import_fundsxml.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4e0b04..19cc72b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -128,50 +128,62 @@ jobs: fi echo "tamper correctly detected" - - name: DB integration - multi-fund round-trip in 4 languages + - name: DB integration - multi-fund import+export in 4 languages run: | set -e python3 -m pip install --quiet lxml tools/fetch-tools.sh # sqlite-jdbc -> .lib/ FX=FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml MX=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml + DOC=FUNDSXML_MULTI_1 EQ="python3 Database_Integration/tools/xml_equiv.py" XSD=".schema-cache/4.2.9/FundsXML.xsd" # The multi-fund fixture must itself be schema-valid. xmllint --noout --nonet --schema "$XSD" "$FX" + # Each language: separate import then export, then prove + # exported file == input file (and is schema-valid). + # Python - python3 Database_Integration/python/fundsxml_db.py roundtrip "$FX" py.xml + python3 Database_Integration/python/import_fundsxml.py py.db "$FX" + python3 Database_Integration/python/export_fundsxml.py py.db "$DOC" py.xml xmllint --noout --nonet --schema "$XSD" py.xml $EQ "$FX" py.xml # Java (sqlite-jdbc) CP=.lib/sqlite-jdbc-3.46.1.3.jar - javac -cp "$CP" -d /tmp/jdb Database_Integration/java/FundsXmlDb.java + javac -cp "$CP" -d /tmp/jdb \ + Database_Integration/java/ImportFundsXml.java \ + Database_Integration/java/ExportFundsXml.java + java --enable-native-access=ALL-UNNAMED -cp "$CP:/tmp/jdb" \ + ImportFundsXml java.db "$FX" java --enable-native-access=ALL-UNNAMED -cp "$CP:/tmp/jdb" \ - FundsXmlDb roundtrip "$FX" java.xml + ExportFundsXml java.db "$DOC" java.xml xmllint --noout --nonet --schema "$XSD" java.xml $EQ "$FX" java.xml # JavaScript (sql.js) ( cd Database_Integration/javascript && npm install --no-audit --no-fund ) - node Database_Integration/javascript/fundsxml_db.mjs roundtrip "$FX" js.xml + node Database_Integration/javascript/import_fundsxml.mjs js.db "$FX" + node Database_Integration/javascript/export_fundsxml.mjs js.db "$DOC" js.xml xmllint --noout --nonet --schema "$XSD" js.xml $EQ "$FX" js.xml - # C# (.NET) - dotnet run --project Database_Integration/csharp -- roundtrip "$FX" cs.xml + # C# (.NET): separate import and export projects + dotnet run --project Database_Integration/csharp/import -- cs.db "$FX" + dotnet run --project Database_Integration/csharp/export -- cs.db "$DOC" cs.xml xmllint --noout --nonet --schema "$XSD" cs.xml $EQ "$FX" cs.xml # Cross-language: every export identical to Python's. $EQ py.xml java.xml && $EQ py.xml js.xml && $EQ py.xml cs.xml - # Single-fund sample still round-trips (lossy -> XSD-valid only). - python3 Database_Integration/python/fundsxml_db.py roundtrip "$MX" mx.xml + # Single-fund sample still imports+exports (lossy -> XSD-valid only). + python3 Database_Integration/python/import_fundsxml.py mx.db "$MX" + python3 Database_Integration/python/export_fundsxml.py mx.db FUNDSXML_FILE_1 mx.xml xmllint --noout --nonet --schema "$XSD" mx.xml - echo "multi-fund round-trip equivalent in python/java/javascript/csharp" + echo "multi-fund import+export equivalent in python/java/javascript/csharp" - name: Large-file - streaming aggregate / split / delta (constant memory) run: | diff --git a/Database_Integration/README.md b/Database_Integration/README.md index 1a8be84..b0ca4d7 100644 --- a/Database_Integration/README.md +++ b/Database_Integration/README.md @@ -2,27 +2,28 @@ ![python](https://img.shields.io/badge/Python-verified-brightgreen) ![java](https://img.shields.io/badge/Java-verified-brightgreen) ![node](https://img.shields.io/badge/JavaScript-verified-brightgreen) ![csharp](https://img.shields.io/badge/C%23-verified-brightgreen) ![multi-fund](https://img.shields.io/badge/multi--node-yes-blue) -Standalone, copy-me examples that move data **both directions** between FundsXML -and a relational database, in the language you already use: +Standalone, copy-me examples in the language you already use. **Import and +export are separate programs** — one shows FundsXML → DB, the other DB → +FundsXML — so you can lift exactly the direction you need: -| Language | File | DB driver | Verified | -|----------|------|-----------|----------| -| Python | [`python/fundsxml_db.py`](python/fundsxml_db.py) | stdlib `sqlite3` | ✅ locally | -| Java | [`java/FundsXmlDb.java`](java/FundsXmlDb.java) | `sqlite-jdbc` (native `javax.xml`, no JAXB) | ✅ locally | -| JavaScript | [`javascript/fundsxml_db.mjs`](javascript/fundsxml_db.mjs) | `sql.js` (pure-WASM) | ✅ locally | -| C# / .NET | [`csharp/FundsXmlDb.cs`](csharp/FundsXmlDb.cs) | `Microsoft.Data.Sqlite` | ✅ locally + CI | +| Language | Import (FundsXML → DB) | Export (DB → FundsXML) | DB driver | Verified | +|----------|------------------------|------------------------|-----------|----------| +| Python | [`python/import_fundsxml.py`](python/import_fundsxml.py) | [`python/export_fundsxml.py`](python/export_fundsxml.py) | stdlib `sqlite3` | ✅ locally | +| Java | [`java/ImportFundsXml.java`](java/ImportFundsXml.java) | [`java/ExportFundsXml.java`](java/ExportFundsXml.java) | `sqlite-jdbc` (native `javax.xml`, no JAXB) | ✅ locally | +| JavaScript | [`javascript/import_fundsxml.mjs`](javascript/import_fundsxml.mjs) | [`javascript/export_fundsxml.mjs`](javascript/export_fundsxml.mjs) | `sql.js` (pure-WASM) | ✅ locally | +| C# / .NET | [`csharp/import/`](csharp/import/) | [`csharp/export/`](csharp/export/) | `Microsoft.Data.Sqlite` | ✅ locally + CI | -Every program is **self-contained**, heavily commented as a teaching artifact -(read one top-to-bottom and reimplement the pattern), and exposes the **same -CLI**: +Every program is **self-contained** (one file / one project, its own copy of +the small helper maps — nothing shared to import) and heavily commented as a +teaching artifact. The CLI is uniform across languages: ``` - init create the schema - import FundsXML -> rows - export rows -> FundsXML - roundtrip import then export through the DB + FundsXML -> rows (creates the schema) + rows -> FundsXML ``` +The round-trip is then simply: run import, then run export, then compare. + ## Relational model — multi-node [`ddl/schema.sql`](ddl/schema.sql): `document → fund → portfolio → position`, @@ -54,27 +55,37 @@ XSD-validity is asserted. ## Run +Each example is run as **import, then export** (the round-trip = both, then +compare). `DOC` is the document id the import prints. + ```bash tools/fetch-schema.sh 4.2.9 # XSD for validation FX=FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml +DOC=FUNDSXML_MULTI_1 # Python -python3 Database_Integration/python/fundsxml_db.py roundtrip "$FX" out.xml +python3 Database_Integration/python/import_fundsxml.py fx.db "$FX" +python3 Database_Integration/python/export_fundsxml.py fx.db "$DOC" out.xml # Java (sqlite-jdbc fetched into .lib/ by tools/fetch-tools.sh) tools/fetch-tools.sh CP=.lib/sqlite-jdbc-3.46.1.3.jar -javac -cp "$CP" -d /tmp/db Database_Integration/java/FundsXmlDb.java -java --enable-native-access=ALL-UNNAMED -cp "$CP:/tmp/db" FundsXmlDb roundtrip "$FX" out.xml +javac -cp "$CP" -d /tmp/db \ + Database_Integration/java/ImportFundsXml.java \ + Database_Integration/java/ExportFundsXml.java +java --enable-native-access=ALL-UNNAMED -cp "$CP:/tmp/db" ImportFundsXml fx.db "$FX" +java --enable-native-access=ALL-UNNAMED -cp "$CP:/tmp/db" ExportFundsXml fx.db "$DOC" out.xml # JavaScript ( cd Database_Integration/javascript && npm install ) -node Database_Integration/javascript/fundsxml_db.mjs roundtrip "$FX" out.xml +node Database_Integration/javascript/import_fundsxml.mjs fx.db "$FX" +node Database_Integration/javascript/export_fundsxml.mjs fx.db "$DOC" out.xml # C# -dotnet run --project Database_Integration/csharp -- roundtrip "$FX" out.xml +dotnet run --project Database_Integration/csharp/import -- fx.db "$FX" +dotnet run --project Database_Integration/csharp/export -- fx.db "$DOC" out.xml -# prove it: identical to the input, and schema-valid +# prove it: exported file == input file, and schema-valid python3 Database_Integration/tools/xml_equiv.py "$FX" out.xml xmllint --noout --schema .schema-cache/4.2.9/FundsXML.xsd out.xml ``` diff --git a/Database_Integration/csharp/FundsXmlDb.cs b/Database_Integration/csharp/export/ExportFundsXml.cs similarity index 55% rename from Database_Integration/csharp/FundsXmlDb.cs rename to Database_Integration/csharp/export/ExportFundsXml.cs index 9a8f5d7..19d4b72 100644 --- a/Database_Integration/csharp/FundsXmlDb.cs +++ b/Database_Integration/csharp/export/ExportFundsXml.cs @@ -1,37 +1,29 @@ // ============================================================================= -// FundsXML <-> relational database — runnable reference (C# / .NET, SQLite). +// EXPORT — relational database -> FundsXML file (C# / .NET, SQLite). // -// PURPOSE -// Standalone, copy-me example of BOTH directions (import a FundsXML file into -// the relational schema, export it back). Over-commented on purpose so it -// doubles as documentation for your own implementation. +// Standalone, copy-me example of ONE direction (DB -> FundsXML). The reverse +// is a separate project, ../import/. Over-commented as documentation. // -// DB SCHEMA ../ddl/schema.sql (document -> fund -> portfolio -> position; -// share_class per fund; asset document-scoped). +// DB SCHEMA ../../ddl/schema.sql (already populated by ../import/). // // RUN -// dotnet run --project Database_Integration/csharp -- \ -// roundtrip FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml out.xml -// commands: init | import | export | -// roundtrip +// dotnet run --project Database_Integration/csharp/import -- fx.db some.xml +// dotnet run --project Database_Integration/csharp/export -- \ +// fx.db FUNDSXML_MULTI_1 out.xml // -// DEPENDENCIES -// Microsoft.Data.Sqlite (bundles native SQLite) + System.Xml in the BCL. -// No XSD data-binding: System.Xml DOM/XPath keeps it small and -// version-tolerant (FundsXML 4.x is backward compatible). +// DEPENDENCIES Microsoft.Data.Sqlite + System.Xml (BCL). // -// FUNDSXML ASSUMPTIONS -// * No XML namespace -> bare element names in XPath. -// * Many //: all iterated; 1-based *_seq columns -// preserve order so the round-trip compares equal (../tools/xml_equiv.py). -// * Positions link to AssetMasterData by shared -> `asset` is -// document-scoped. -// * Export normalized to the 4.2.9 schema URL; constants the model does not -// store (TotalAssetNature=OFFICIAL, Price ActionCode=C / PriceNature= -// OFFICIAL) are reproduced verbatim. -// -// SECURITY -// XmlReaderSettings: DtdProcessing.Prohibit + XmlResolver=null (no XXE). +// FUNDSXML NOTES +// * No XML namespace -> plain element names. +// * xsi:noNamespaceSchemaLocation MUST be created in the XMLSchema-instance +// namespace (a plain SetAttribute drops the prefix and a validator then +// rejects it) -> CreateAttribute("xsi", ..., XsiNs). +// * Constants the model does not store (TotalAssetNature=OFFICIAL, Price +// ActionCode=C / PriceNature=OFFICIAL) reproduced verbatim so the +// round-trip compares equal (../../tools/xml_equiv.py, always paired with +// XSD validation). +// * ORDER BY the 1-based *_seq columns reproduces the original order of +// multiple funds / portfolios / positions. // ============================================================================= using System; using System.Globalization; @@ -39,12 +31,12 @@ using System.Xml; using Microsoft.Data.Sqlite; -internal static class FundsXmlDb +internal static class ExportFundsXml { const string SchemaUrl = "https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd"; + const string XsiNs = "http://www.w3.org/2001/XMLSchema-instance"; - // Position instrument-class element + its mandatory quantity child. static readonly System.Collections.Generic.HashSet PositionKinds = new() { "Equity", "Bond", "ShareClass", "Warrant", "Certificate", "Option", "Future", "FXForward", "Swap", "Repo", "RealEstate", @@ -57,17 +49,6 @@ internal static class FundsXmlDb static string Inv(double v) => v.ToString("0.00", CultureInfo.InvariantCulture); - // ---- helpers ----------------------------------------------------------- - static string? T(XmlNode ctx, string xpath) - { - var n = ctx.SelectSingleNode(xpath); - return string.IsNullOrEmpty(n?.InnerText) ? null : n!.InnerText; - } - - static object DbNum(string? s) => - string.IsNullOrWhiteSpace(s) ? DBNull.Value - : double.Parse(s, CultureInfo.InvariantCulture); - static XmlElement El(XmlDocument doc, XmlNode parent, string tag, string? text = null) { @@ -77,157 +58,6 @@ static XmlElement El(XmlDocument doc, XmlNode parent, string tag, return e; } - static SqliteConnection Open(string db) - { - var c = new SqliteConnection($"Data Source={db}"); - c.Open(); - using var pragma = c.CreateCommand(); - pragma.CommandText = "PRAGMA foreign_keys = ON"; - pragma.ExecuteNonQuery(); - return c; - } - - static void Exec(SqliteConnection c, string sql) - { - using var cmd = c.CreateCommand(); - cmd.CommandText = sql; - cmd.ExecuteNonQuery(); - } - - // Bind helper: positional @p0,@p1,... mirrors the other languages' "?". - static void Run(SqliteConnection c, string sql, params object?[] ps) - { - using var cmd = c.CreateCommand(); - cmd.CommandText = sql; - for (int i = 0; i < ps.Length; i++) - cmd.Parameters.AddWithValue($"@p{i}", ps[i] ?? DBNull.Value); - cmd.ExecuteNonQuery(); - } - - static XmlDocument ParseSecure(string xml) - { - var settings = new XmlReaderSettings - { - DtdProcessing = DtdProcessing.Prohibit, // reject DOCTYPE (XXE) - XmlResolver = null // no external entities - }; - var doc = new XmlDocument(); - using var r = XmlReader.Create(xml, settings); - doc.Load(r); - return doc; - } - - static string DdlPath() - { - // Works whether run from repo root or the csharp/ project dir. - foreach (var p in new[] { Path.Combine("Database_Integration", "ddl", - "schema.sql"), Path.Combine("..", "ddl", "schema.sql") }) - if (File.Exists(p)) return p; - throw new FileNotFoundException("ddl/schema.sql not found"); - } - - static void Init(string db) - { - // Strip "--" line comments (no string literals in the DDL) then run - // each ";"-separated statement. - var lines = File.ReadAllLines(DdlPath()); - var sb = new System.Text.StringBuilder(); - foreach (var line in lines) - { - int cut = line.IndexOf("--", StringComparison.Ordinal); - sb.AppendLine(cut >= 0 ? line[..cut] : line); - } - using var c = Open(db); - foreach (var stmt in sb.ToString().Split(';')) - if (stmt.Trim().Length > 0) Exec(c, stmt); - } - - // ---- import : FundsXML -> rows ----------------------------------------- - static string DoImport(string db, string xml) - { - var doc = ParseSecure(xml); - var cd = doc.SelectSingleNode("/FundsXML4/ControlData")!; - var docId = T(cd, "UniqueDocumentID")!; - - using var c = Open(db); - using var tx = c.BeginTransaction(); - - Run(c, "INSERT INTO document VALUES (@p0,@p1,@p2,@p3,@p4,@p5,@p6,@p7,@p8)", - docId, T(cd, "DocumentGenerated"), T(cd, "Version"), - T(cd, "ContentDate"), T(cd, "DataOperation"), - T(cd, "DataSupplier/SystemCountry"), T(cd, "DataSupplier/Short"), - T(cd, "DataSupplier/Name"), T(cd, "DataSupplier/Type")); - - var funds = doc.SelectNodes("/FundsXML4/Funds/Fund")!; - for (int fi = 0; fi < funds.Count; fi++) - { - var fund = funds[fi]!; - int fundSeq = fi + 1; // 1-based doc order - var ccy = T(fund, "Currency")!; - var tav = fund.SelectSingleNode( - "FundDynamicData/TotalAssetValues/TotalAssetValue")!; - Run(c, "INSERT INTO fund VALUES (@p0,@p1,@p2,@p3,@p4,@p5,@p6,@p7)", - docId, fundSeq, T(fund, "Identifiers/LEI"), - T(fund, "Names/OfficialName"), ccy, - T(fund, "SingleFundFlag"), T(tav, "NavDate"), - double.Parse(T(tav, - $"TotalNetAssetValue/Amount[@ccy='{ccy}']")!, - CultureInfo.InvariantCulture)); - - var scs = fund.SelectNodes("SingleFund/ShareClasses/ShareClass")!; - foreach (XmlNode sc in scs) - Run(c, "INSERT INTO share_class VALUES (@p0,@p1,@p2,@p3,@p4,@p5,@p6,@p7)", - docId, fundSeq, T(sc, "Identifiers/ISIN"), - T(sc, "Names/OfficialName"), T(sc, "Currency"), - DbNum(T(sc, "Prices/Price/NavPrice")), - DbNum(T(sc, $"TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount[@ccy='{ccy}']")), - DbNum(T(sc, "TotalAssetValues/TotalAssetValue/SharesOutstanding"))); - - var ports = fund.SelectNodes( - "FundDynamicData/Portfolios/Portfolio")!; - for (int pi = 0; pi < ports.Count; pi++) - { - var port = ports[pi]!; - int portSeq = pi + 1; - Run(c, "INSERT INTO portfolio VALUES (@p0,@p1,@p2,@p3)", - docId, fundSeq, portSeq, T(port, "NavDate")); - var poss = port.SelectNodes("Positions/Position")!; - for (int qi = 0; qi < poss.Count; qi++) - { - var pos = poss[qi]!; - string? kind = null; - foreach (XmlNode ch in pos.ChildNodes) - if (ch.NodeType == XmlNodeType.Element - && PositionKinds.Contains(ch.Name)) - { kind = ch.Name; break; } - object qty = (kind != null && QtyElem.ContainsKey(kind)) - ? DbNum(T(pos, $"{kind}/{QtyElem[kind]}")) - : DBNull.Value; - Run(c, "INSERT INTO position VALUES (@p0,@p1,@p2,@p3,@p4,@p5,@p6,@p7,@p8,@p9,@p10)", - docId, fundSeq, portSeq, qi + 1, - T(pos, "UniqueID"), T(pos, "Identifiers/ISIN"), - T(pos, "Currency"), - double.Parse(T(pos, $"TotalValue/Amount[@ccy='{ccy}']")!, - CultureInfo.InvariantCulture), - double.Parse(T(pos, "TotalPercentage")!, - CultureInfo.InvariantCulture), - kind, qty); - } - } - } - - var assets = doc.SelectNodes("/FundsXML4/AssetMasterData/Asset")!; - foreach (XmlNode a in assets) - Run(c, "INSERT INTO asset VALUES (@p0,@p1,@p2,@p3,@p4,@p5,@p6)", - docId, T(a, "UniqueID"), T(a, "Identifiers/ISIN"), - T(a, "Name"), T(a, "AssetType"), T(a, "Currency"), - T(a, "Country")); - - tx.Commit(); - return docId; - } - - // ---- export : rows -> FundsXML ---------------------------------------- static SqliteDataReader Q(SqliteConnection c, string sql, params object?[] ps) { @@ -237,7 +67,6 @@ static SqliteDataReader Q(SqliteConnection c, string sql, cmd.Parameters.AddWithValue($"@p{i}", ps[i] ?? DBNull.Value); return cmd.ExecuteReader(); } - static string? S(SqliteDataReader r, string col) => r.IsDBNull(r.GetOrdinal(col)) ? null : r.GetString(r.GetOrdinal(col)); static double D(SqliteDataReader r, string col) => @@ -246,16 +75,22 @@ static double D(SqliteDataReader r, string col) => static bool Null(SqliteDataReader r, string col) => r.IsDBNull(r.GetOrdinal(col)); - static void Export(string db, string docId, string outPath) + static int Main(string[] args) { - using var c = Open(db); + if (args.Length != 3) + { + Console.Error.WriteLine( + "usage: ExportFundsXml "); + return 2; + } + string db = args[0], docId = args[1], outPath = args[2]; + + using var c = new SqliteConnection($"Data Source={db}"); + c.Open(); + var doc = new XmlDocument(); var root = doc.CreateElement("FundsXML4"); - // xsi:noNamespaceSchemaLocation must really live in the - // XMLSchema-instance namespace, otherwise a validator rejects it - // ("attribute 'noNamespaceSchemaLocation' is not allowed"). Create it - // as a properly namespaced attribute (this also emits xmlns:xsi). - const string XsiNs = "http://www.w3.org/2001/XMLSchema-instance"; + // Must really be in the xsi namespace, else a validator rejects it. var sl = doc.CreateAttribute("xsi", "noNamespaceSchemaLocation", XsiNs); sl.Value = SchemaUrl; root.Attributes.Append(sl); @@ -459,48 +294,11 @@ static void Export(string db, string docId, string outPath) } } - var settings = new XmlWriterSettings + var ws = new XmlWriterSettings { Indent = true, Encoding = new System.Text.UTF8Encoding(false) }; - using var w = XmlWriter.Create(outPath, settings); + using var w = XmlWriter.Create(outPath, ws); doc.Save(w); - } - - static int Main(string[] args) - { - if (args.Length == 0) - { - Console.Error.WriteLine("usage: init|import|export|roundtrip ..."); - return 2; - } - switch (args[0]) - { - case "init": - Init(args[1]); - break; - case "import": - Console.WriteLine("imported " + DoImport(args[1], args[2])); - break; - case "export": - Export(args[1], args[2], args[3]); - Console.WriteLine("wrote " + args[3]); - break; - case "roundtrip": - { - // import then export THROUGH the DB (the required test). - var tmp = Path.GetTempFileName(); - File.Delete(tmp); - Init(tmp); - var id = DoImport(tmp, args[1]); - Export(tmp, id, args[2]); - File.Delete(tmp); - Console.WriteLine($"round-trip ok: {args[1]} -> DB -> " - + $"{args[2]} (doc {id})"); - break; - } - default: - Console.Error.WriteLine("unknown: " + args[0]); - return 2; - } + Console.WriteLine("wrote " + outPath); return 0; } } diff --git a/Database_Integration/csharp/FundsXmlDb.csproj b/Database_Integration/csharp/export/ExportFundsXml.csproj similarity index 50% rename from Database_Integration/csharp/FundsXmlDb.csproj rename to Database_Integration/csharp/export/ExportFundsXml.csproj index 63a3ac2..4ca9c9f 100644 --- a/Database_Integration/csharp/FundsXmlDb.csproj +++ b/Database_Integration/csharp/export/ExportFundsXml.csproj @@ -1,14 +1,14 @@ - + Exe net8.0 enable disable - FundsXmlDb - FundsXml.DbExample + ExportFundsXml + FundsXml.DbExample.Export diff --git a/Database_Integration/csharp/import/ImportFundsXml.cs b/Database_Integration/csharp/import/ImportFundsXml.cs new file mode 100644 index 0000000..974041a --- /dev/null +++ b/Database_Integration/csharp/import/ImportFundsXml.cs @@ -0,0 +1,185 @@ +// ============================================================================= +// IMPORT — FundsXML file -> relational database (C# / .NET, SQLite). +// +// Standalone, copy-me example of ONE direction (FundsXML -> DB). The reverse +// is a separate project, ../export/. Over-commented as documentation. +// +// DB SCHEMA ../../ddl/schema.sql (document -> fund -> portfolio -> position; +// share_class per fund; asset document-scoped). Creates the schema in a +// fresh SQLite DB, then loads the file. +// +// RUN +// dotnet run --project Database_Integration/csharp/import -- \ +// fx.db FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml +// dotnet run --project Database_Integration/csharp/export -- \ +// fx.db FUNDSXML_MULTI_1 out.xml +// +// DEPENDENCIES Microsoft.Data.Sqlite + System.Xml (BCL). No XSD binding. +// +// FUNDSXML ASSUMPTIONS +// * No XML namespace -> bare element names in XPath. +// * Many //: all iterated; 1-based *_seq columns +// preserve order so the separate export reproduces the original document. +// * Positions link to AssetMasterData by shared -> `asset` is +// document-scoped. +// +// SECURITY XmlReaderSettings: DtdProcessing.Prohibit + XmlResolver=null (XXE). +// ============================================================================= +using System; +using System.Globalization; +using System.IO; +using System.Xml; +using Microsoft.Data.Sqlite; + +internal static class ImportFundsXml +{ + static readonly System.Collections.Generic.HashSet PositionKinds = + new() { "Equity", "Bond", "ShareClass", "Warrant", "Certificate", + "Option", "Future", "FXForward", "Swap", "Repo", "RealEstate", + "CallMoney", "Account", "Generic" }; + static readonly System.Collections.Generic.Dictionary QtyElem = + new() { ["Equity"] = "Units", ["Warrant"] = "Units", + ["Certificate"] = "Units", ["Bond"] = "Nominal", + ["ShareClass"] = "Shares", ["Option"] = "Contracts", + ["Future"] = "Contracts" }; + + static string? T(XmlNode ctx, string xpath) + { + var n = ctx.SelectSingleNode(xpath); + return string.IsNullOrEmpty(n?.InnerText) ? null : n!.InnerText; + } + static object DbNum(string? s) => + string.IsNullOrWhiteSpace(s) ? DBNull.Value + : double.Parse(s, CultureInfo.InvariantCulture); + + static void Run(SqliteConnection c, string sql, params object?[] ps) + { + using var cmd = c.CreateCommand(); + cmd.CommandText = sql; + for (int i = 0; i < ps.Length; i++) + cmd.Parameters.AddWithValue($"@p{i}", ps[i] ?? DBNull.Value); + cmd.ExecuteNonQuery(); + } + + static string DdlPath() + { + foreach (var p in new[] { + Path.Combine("Database_Integration", "ddl", "schema.sql"), + Path.Combine("..", "..", "ddl", "schema.sql") }) + if (File.Exists(p)) return p; + throw new FileNotFoundException("ddl/schema.sql not found"); + } + + static int Main(string[] args) + { + if (args.Length != 2) + { + Console.Error.WriteLine( + "usage: ImportFundsXml "); + return 2; + } + string db = args[0], xml = args[1]; + + // ---- parse the FundsXML file (hardened) --------------------------- + var settings = new XmlReaderSettings + { + DtdProcessing = DtdProcessing.Prohibit, + XmlResolver = null + }; + var doc = new XmlDocument(); + using (var r = XmlReader.Create(xml, settings)) doc.Load(r); + + using var c = new SqliteConnection($"Data Source={db}"); + c.Open(); + Run(c, "PRAGMA foreign_keys = ON"); + + // ---- create the schema (fresh DB) --------------------------------- + var sb = new System.Text.StringBuilder(); + foreach (var line in File.ReadAllLines(DdlPath())) + { + int cut = line.IndexOf("--", StringComparison.Ordinal); + sb.AppendLine(cut >= 0 ? line[..cut] : line); + } + foreach (var stmt in sb.ToString().Split(';')) + if (stmt.Trim().Length > 0) Run(c, stmt); + + using var tx = c.BeginTransaction(); + var cd = doc.SelectSingleNode("/FundsXML4/ControlData")!; + var docId = T(cd, "UniqueDocumentID")!; + + Run(c, "INSERT INTO document VALUES (@p0,@p1,@p2,@p3,@p4,@p5,@p6,@p7,@p8)", + docId, T(cd, "DocumentGenerated"), T(cd, "Version"), + T(cd, "ContentDate"), T(cd, "DataOperation"), + T(cd, "DataSupplier/SystemCountry"), T(cd, "DataSupplier/Short"), + T(cd, "DataSupplier/Name"), T(cd, "DataSupplier/Type")); + + var funds = doc.SelectNodes("/FundsXML4/Funds/Fund")!; + for (int fi = 0; fi < funds.Count; fi++) + { + var fund = funds[fi]!; + int fundSeq = fi + 1; // 1-based doc order + var ccy = T(fund, "Currency")!; + var tav = fund.SelectSingleNode( + "FundDynamicData/TotalAssetValues/TotalAssetValue")!; + Run(c, "INSERT INTO fund VALUES (@p0,@p1,@p2,@p3,@p4,@p5,@p6,@p7)", + docId, fundSeq, T(fund, "Identifiers/LEI"), + T(fund, "Names/OfficialName"), ccy, + T(fund, "SingleFundFlag"), T(tav, "NavDate"), + double.Parse(T(tav, + $"TotalNetAssetValue/Amount[@ccy='{ccy}']")!, + CultureInfo.InvariantCulture)); + + foreach (XmlNode sc in fund.SelectNodes( + "SingleFund/ShareClasses/ShareClass")!) + Run(c, "INSERT INTO share_class VALUES (@p0,@p1,@p2,@p3,@p4,@p5,@p6,@p7)", + docId, fundSeq, T(sc, "Identifiers/ISIN"), + T(sc, "Names/OfficialName"), T(sc, "Currency"), + DbNum(T(sc, "Prices/Price/NavPrice")), + DbNum(T(sc, $"TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount[@ccy='{ccy}']")), + DbNum(T(sc, "TotalAssetValues/TotalAssetValue/SharesOutstanding"))); + + var ports = fund.SelectNodes( + "FundDynamicData/Portfolios/Portfolio")!; + for (int pi = 0; pi < ports.Count; pi++) + { + var port = ports[pi]!; + int portSeq = pi + 1; + Run(c, "INSERT INTO portfolio VALUES (@p0,@p1,@p2,@p3)", + docId, fundSeq, portSeq, T(port, "NavDate")); + var poss = port.SelectNodes("Positions/Position")!; + for (int qi = 0; qi < poss.Count; qi++) + { + var pos = poss[qi]!; + string? kind = null; + foreach (XmlNode ch in pos.ChildNodes) + if (ch.NodeType == XmlNodeType.Element + && PositionKinds.Contains(ch.Name)) + { kind = ch.Name; break; } + object qty = (kind != null && QtyElem.ContainsKey(kind)) + ? DbNum(T(pos, $"{kind}/{QtyElem[kind]}")) + : DBNull.Value; + Run(c, "INSERT INTO position VALUES (@p0,@p1,@p2,@p3,@p4,@p5,@p6,@p7,@p8,@p9,@p10)", + docId, fundSeq, portSeq, qi + 1, + T(pos, "UniqueID"), T(pos, "Identifiers/ISIN"), + T(pos, "Currency"), + double.Parse(T(pos, $"TotalValue/Amount[@ccy='{ccy}']")!, + CultureInfo.InvariantCulture), + double.Parse(T(pos, "TotalPercentage")!, + CultureInfo.InvariantCulture), + kind, qty); + } + } + } + + foreach (XmlNode a in doc.SelectNodes( + "/FundsXML4/AssetMasterData/Asset")!) + Run(c, "INSERT INTO asset VALUES (@p0,@p1,@p2,@p3,@p4,@p5,@p6)", + docId, T(a, "UniqueID"), T(a, "Identifiers/ISIN"), + T(a, "Name"), T(a, "AssetType"), T(a, "Currency"), + T(a, "Country")); + + tx.Commit(); + Console.WriteLine("imported document_id: " + docId); + return 0; + } +} diff --git a/Database_Integration/csharp/import/ImportFundsXml.csproj b/Database_Integration/csharp/import/ImportFundsXml.csproj new file mode 100644 index 0000000..b53feca --- /dev/null +++ b/Database_Integration/csharp/import/ImportFundsXml.csproj @@ -0,0 +1,17 @@ + + + + Exe + net8.0 + enable + disable + ImportFundsXml + FundsXml.DbExample.Import + + + + + diff --git a/Database_Integration/java/ExportFundsXml.java b/Database_Integration/java/ExportFundsXml.java new file mode 100644 index 0000000..3dc977e --- /dev/null +++ b/Database_Integration/java/ExportFundsXml.java @@ -0,0 +1,248 @@ +// ============================================================================= +// EXPORT — relational database -> FundsXML file (Java, SQLite via JDBC). +// +// Standalone, copy-me example of ONE direction (DB -> FundsXML). The reverse +// is a separate program, ImportFundsXml.java. Over-commented as documentation. +// +// DB SCHEMA ../ddl/schema.sql (already populated by ImportFundsXml). +// +// RUN +// CP=.lib/sqlite-jdbc-3.46.1.3.jar +// javac -cp "$CP" -d /tmp/db Database_Integration/java/ExportFundsXml.java +// java --enable-native-access=ALL-UNNAMED -cp "$CP:/tmp/db" \ +// ExportFundsXml fx.db FUNDSXML_MULTI_1 out.xml +// +// DEPENDENCIES org.xerial:sqlite-jdbc + the JDK (native javax.xml, no JAXB). +// +// FUNDSXML NOTES +// * No XML namespace -> plain element names. +// * Output normalized to the 4.2.9 schema URL; constants the model does not +// store (TotalAssetNature=OFFICIAL, Price ActionCode=C / PriceNature= +// OFFICIAL) reproduced verbatim so the round-trip compares equal +// (../tools/xml_equiv.py, always paired with XSD validation). +// * ORDER BY the 1-based *_seq columns reproduces the original order of +// multiple funds / portfolios / positions. +// ============================================================================= +import java.io.FileOutputStream; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.Locale; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +public class ExportFundsXml { + + static final String SCHEMA_URL = + "https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd"; + static final String XSI = "http://www.w3.org/2001/XMLSchema-instance"; + static final java.util.Set POSITION_KINDS = java.util.Set.of( + "Equity", "Bond", "ShareClass", "Warrant", "Certificate", "Option", + "Future", "FXForward", "Swap", "Repo", "RealEstate", "CallMoney", + "Account", "Generic"); + static final java.util.Map QTY_ELEM = java.util.Map.of( + "Equity", "Units", "Warrant", "Units", "Certificate", "Units", + "Bond", "Nominal", "ShareClass", "Shares", + "Option", "Contracts", "Future", "Contracts"); + + static String f2(double v) { return String.format(Locale.ROOT, "%.2f", v); } + + /** Append text to parent (the one XML-build primitive). */ + static Element el(Document doc, org.w3c.dom.Node parent, String tag, + String text) { + Element e = doc.createElement(tag); + if (text != null) e.setTextContent(text); + parent.appendChild(e); + return e; + } + + public static void main(String[] args) throws Exception { + if (args.length != 3) { + System.err.println( + "usage: ExportFundsXml "); + System.exit(2); + } + String db = args[0], docId = args[1], out = args[2]; + + Document doc = DocumentBuilderFactory.newInstance() + .newDocumentBuilder().newDocument(); + Element root = doc.createElement("FundsXML4"); + root.setAttribute("xmlns:xsi", XSI); + root.setAttribute("xsi:noNamespaceSchemaLocation", SCHEMA_URL); + doc.appendChild(root); + + try (Connection c = DriverManager.getConnection("jdbc:sqlite:" + db)) { + ResultSet d = c.createStatement().executeQuery( + "SELECT * FROM document WHERE document_id='" + docId + "'"); + if (!d.next()) throw new RuntimeException("no document " + docId); + + Element cd = el(doc, root, "ControlData", null); + el(doc, cd, "UniqueDocumentID", d.getString("document_id")); + // Regenerate timestamp; xml_equiv.py ignores its value. + String gen = d.getString("generated"); + el(doc, cd, "DocumentGenerated", + gen != null ? gen : "2025-10-02T00:00:00"); + String ver = d.getString("version"); + if (ver != null) el(doc, cd, "Version", ver); // none for 4.0.0 + el(doc, cd, "ContentDate", d.getString("content_date")); + Element ds = el(doc, cd, "DataSupplier", null); + el(doc, ds, "SystemCountry", d.getString("supplier_country")); + el(doc, ds, "Short", d.getString("supplier_short")); + el(doc, ds, "Name", d.getString("supplier_name")); + el(doc, ds, "Type", d.getString("supplier_type")); + el(doc, cd, "DataOperation", d.getString("data_operation")); + + Element fundsEl = el(doc, root, "Funds", null); + PreparedStatement fps = c.prepareStatement( + "SELECT * FROM fund WHERE document_id=? ORDER BY fund_seq"); + fps.setString(1, docId); + ResultSet fr = fps.executeQuery(); + while (fr.next()) { + int fundSeq = fr.getInt("fund_seq"); + String ccy = fr.getString("currency"); + String navDate = fr.getString("nav_date"); + Element fund = el(doc, fundsEl, "Fund", null); + if (fr.getString("lei") != null) + el(doc, el(doc, fund, "Identifiers", null), "LEI", + fr.getString("lei")); + el(doc, el(doc, fund, "Names", null), "OfficialName", + fr.getString("official_name")); + el(doc, fund, "Currency", ccy); + if (fr.getString("single_fund_flag") != null) + el(doc, fund, "SingleFundFlag", + fr.getString("single_fund_flag")); + + Element fdd = el(doc, fund, "FundDynamicData", null); + Element tav = el(doc, el(doc, el(doc, fdd, + "TotalAssetValues", null), "TotalAssetValue", null), + "NavDate", navDate); + Element tavp = (Element) tav.getParentNode(); + el(doc, tavp, "TotalAssetNature", "OFFICIAL"); + Element amt = el(doc, el(doc, tavp, "TotalNetAssetValue", + null), "Amount", f2(fr.getDouble("total_nav"))); + amt.setAttribute("ccy", ccy); + + Element ports = el(doc, fdd, "Portfolios", null); + PreparedStatement pps = c.prepareStatement( + "SELECT * FROM portfolio WHERE document_id=? AND fund_seq=?" + + " ORDER BY portfolio_seq"); + pps.setString(1, docId); pps.setInt(2, fundSeq); + ResultSet pr = pps.executeQuery(); + while (pr.next()) { + int portSeq = pr.getInt("portfolio_seq"); + Element pe = el(doc, ports, "Portfolio", null); + el(doc, pe, "NavDate", pr.getString("nav_date")); + Element poss = el(doc, pe, "Positions", null); + PreparedStatement qps = c.prepareStatement( + "SELECT * FROM position WHERE document_id=? AND " + + "fund_seq=? AND portfolio_seq=? ORDER BY position_seq"); + qps.setString(1, docId); qps.setInt(2, fundSeq); + qps.setInt(3, portSeq); + ResultSet qr = qps.executeQuery(); + while (qr.next()) { + Element pos = el(doc, poss, "Position", null); + el(doc, pos, "UniqueID", qr.getString("unique_id")); + if (qr.getString("isin") != null) + el(doc, el(doc, pos, "Identifiers", null), "ISIN", + qr.getString("isin")); + if (qr.getString("currency") != null) + el(doc, pos, "Currency", qr.getString("currency")); + Element tv = el(doc, el(doc, pos, "TotalValue", null), + "Amount", f2(qr.getDouble("value_fund_ccy"))); + tv.setAttribute("ccy", ccy); + el(doc, pos, "TotalPercentage", + f2(qr.getDouble("percentage"))); + String kind = qr.getString("kind"); + if (kind == null || !POSITION_KINDS.contains(kind)) + kind = "Generic"; + Element ke = el(doc, pos, kind, null); + Object q = qr.getObject("kind_qty"); + if (QTY_ELEM.containsKey(kind) && q != null) + el(doc, ke, QTY_ELEM.get(kind), + f2(((Number) q).doubleValue())); + } + } + + PreparedStatement sps = c.prepareStatement( + "SELECT * FROM share_class WHERE document_id=? AND " + + "fund_seq=? ORDER BY isin"); + sps.setString(1, docId); sps.setInt(2, fundSeq); + ResultSet sr = sps.executeQuery(); + Element sce = null; + while (sr.next()) { + if (sce == null) + sce = el(doc, el(doc, fund, "SingleFund", null), + "ShareClasses", null); + Element x = el(doc, sce, "ShareClass", null); + el(doc, el(doc, x, "Identifiers", null), "ISIN", + sr.getString("isin")); + if (sr.getString("official_name") != null) + el(doc, el(doc, x, "Names", null), "OfficialName", + sr.getString("official_name")); + el(doc, x, "Currency", sr.getString("currency")); + Object navp = sr.getObject("nav_price"); + if (navp != null) { + Element pe2 = el(doc, el(doc, x, "Prices", null), + "Price", null); + el(doc, pe2, "ActionCode", "C"); + el(doc, pe2, "NavDate", navDate); + el(doc, pe2, "PriceCurrency", sr.getString("currency")); + el(doc, pe2, "PriceNature", "OFFICIAL"); + el(doc, pe2, "NavPrice", + f2(((Number) navp).doubleValue())); + } + Object navf = sr.getObject("nav_fund_ccy"); + if (navf != null) { + Element t2 = el(doc, el(doc, x, "TotalAssetValues", + null), "TotalAssetValue", null); + el(doc, t2, "NavDate", navDate); + el(doc, t2, "TotalAssetNature", "OFFICIAL"); + Element a2 = el(doc, el(doc, t2, + "TotalNetAssetValue", null), "Amount", + f2(((Number) navf).doubleValue())); + a2.setAttribute("ccy", ccy); + Object so = sr.getObject("shares_outstanding"); + if (so != null) + el(doc, t2, "SharesOutstanding", String.format( + Locale.ROOT, "%.0f", + ((Number) so).doubleValue())); + } + } + } + + PreparedStatement aps = c.prepareStatement( + "SELECT * FROM asset WHERE document_id=? ORDER BY unique_id"); + aps.setString(1, docId); + ResultSet ar = aps.executeQuery(); + Element amd = null; + while (ar.next()) { + if (amd == null) amd = el(doc, root, "AssetMasterData", null); + Element ae = el(doc, amd, "Asset", null); + el(doc, ae, "UniqueID", ar.getString("unique_id")); + if (ar.getString("isin") != null) + el(doc, el(doc, ae, "Identifiers", null), "ISIN", + ar.getString("isin")); + el(doc, ae, "Currency", ar.getString("currency")); + if (ar.getString("country") != null) + el(doc, ae, "Country", ar.getString("country")); + el(doc, ae, "Name", ar.getString("name")); + el(doc, ae, "AssetType", ar.getString("asset_type")); + } + } + + Transformer tr = TransformerFactory.newInstance().newTransformer(); + tr.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); + tr.setOutputProperty(OutputKeys.INDENT, "yes"); + try (FileOutputStream os = new FileOutputStream(out)) { + tr.transform(new DOMSource(doc), new StreamResult(os)); + } + System.out.println("wrote " + out); + } +} diff --git a/Database_Integration/java/FundsXmlDb.java b/Database_Integration/java/FundsXmlDb.java deleted file mode 100644 index 2d83b04..0000000 --- a/Database_Integration/java/FundsXmlDb.java +++ /dev/null @@ -1,493 +0,0 @@ -// ============================================================================= -// FundsXML <-> relational database — runnable reference (Java, SQLite via JDBC). -// -// PURPOSE -// Standalone, copy-me example showing BOTH directions (import a FundsXML file -// into the relational schema, and export it back out). Over-commented on -// purpose: read it once and reimplement the pattern in your own project. -// -// DB SCHEMA -// ../ddl/schema.sql (document -> fund -> portfolio -> position; share_class -// per fund; asset document-scoped). SQLite keeps the example zero-setup. -// -// RUN (the sqlite-jdbc jar is fetched by tools/fetch-tools.sh into .lib/) -// tools/fetch-tools.sh -// CP=.lib/sqlite-jdbc-3.46.1.3.jar -// javac -cp "$CP" -d /tmp/db Database_Integration/java/FundsXmlDb.java -// java -cp "$CP:/tmp/db" FundsXmlDb roundtrip \ -// FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml out.xml -// commands: init | import | export | -// roundtrip -// -// DEPENDENCIES -// org.xerial:sqlite-jdbc (self-contained, bundles native SQLite) + the JDK. -// XML uses native javax.xml DOM/XPath — NO JAXB (the FundsXML schema is huge; -// a thin DOM binding is simpler and version-tolerant). -// -// FUNDSXML ASSUMPTIONS (non-obvious) -// * FundsXML 4.x has NO XML namespace -> XPath uses bare element names. -// * Many , each many , each many : we iterate all -// and store 1-based *_seq ordinals so the export is order-faithful and the -// round-trip compares equal (see ../tools/xml_equiv.py). -// * Positions link to AssetMasterData by shared ; `asset` is -// therefore document-scoped, not per-fund. -// * Export is normalized to the 4.2.9 schema URL. Constants the model does -// not store (TotalAssetNature=OFFICIAL, Price ActionCode=C/PriceNature= -// OFFICIAL) are reproduced verbatim so the round-trip stays equal. -// -// SECURITY -// The DOM parser forbids DOCTYPE/external entities (XXE / billion laughs). -// ============================================================================= -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.Statement; -import java.util.Locale; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathFactory; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - -public class FundsXmlDb { - - static final String SCHEMA_URL = - "https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd"; - static final String XSI = "http://www.w3.org/2001/XMLSchema-instance"; - - // Position instrument-class element + its mandatory quantity child. - // Kinds absent from QTY_ELEM are schema-valid as an empty element. - static final java.util.Set POSITION_KINDS = java.util.Set.of( - "Equity", "Bond", "ShareClass", "Warrant", "Certificate", "Option", - "Future", "FXForward", "Swap", "Repo", "RealEstate", "CallMoney", - "Account", "Generic"); - static final java.util.Map QTY_ELEM = java.util.Map.of( - "Equity", "Units", "Warrant", "Units", "Certificate", "Units", - "Bond", "Nominal", "ShareClass", "Shares", - "Option", "Contracts", "Future", "Contracts"); - - // ---- helpers ----------------------------------------------------------- - static XPath xpath() { return XPathFactory.newInstance().newXPath(); } - - /** First matching descendant's text (bare element names; no namespace). */ - static String t(Node ctx, String expr) throws Exception { - Node n = (Node) xpath().evaluate(expr, ctx, XPathConstants.NODE); - return n == null ? null : n.getTextContent(); - } - - static Double num(String s) { - return (s == null || s.isBlank()) ? null : Double.valueOf(s.trim()); - } - - static String f2(double v) { return String.format(Locale.ROOT, "%.2f", v); } - - /** Append text to parent (the single XML-build primitive). */ - static Element el(Document doc, Node parent, String tag, String text) { - Element e = doc.createElement(tag); - if (text != null) e.setTextContent(text); - parent.appendChild(e); - return e; - } - - static Connection open(String db) throws Exception { - Connection c = DriverManager.getConnection("jdbc:sqlite:" + db); - c.createStatement().execute("PRAGMA foreign_keys = ON"); - return c; - } - - static Document parseSecure(String xml) throws Exception { - DocumentBuilderFactory f = DocumentBuilderFactory.newInstance(); - f.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - f.setFeature("http://xml.org/sax/features/external-general-entities", false); - f.setFeature("http://xml.org/sax/features/external-parameter-entities", false); - f.setExpandEntityReferences(false); - try (FileInputStream in = new FileInputStream(xml)) { - return f.newDocumentBuilder().parse(in); - } - } - - // ---- init -------------------------------------------------------------- - static void init(String db) throws Exception { - Path ddl = Path.of(System.getProperty("user.dir"), - "Database_Integration", "ddl", "schema.sql"); - // Fallback when run from inside Database_Integration/java. - if (!Files.exists(ddl)) - ddl = Path.of("..", "ddl", "schema.sql"); - // Strip "--" line comments first (the DDL has no string literals, so a - // plain cut at "--" is safe) so only real SQL statements remain. - StringBuilder clean = new StringBuilder(); - for (String line : Files.readString(ddl).split("\n")) { - int cut = line.indexOf("--"); - clean.append(cut >= 0 ? line.substring(0, cut) : line).append('\n'); - } - try (Connection c = open(db); Statement st = c.createStatement()) { - for (String stmt : clean.toString().split(";")) { - if (!stmt.strip().isEmpty()) st.execute(stmt); - } - } - } - - // ---- import : FundsXML -> rows ----------------------------------------- - static String doImport(String db, String xml) throws Exception { - Document doc = parseSecure(xml); - XPath xp = xpath(); - Node cd = (Node) xp.evaluate("/FundsXML4/ControlData", doc, - XPathConstants.NODE); - String docId = t(cd, "UniqueDocumentID"); - - try (Connection c = open(db)) { - c.setAutoCommit(false); - - try (PreparedStatement ps = c.prepareStatement( - "INSERT INTO document VALUES (?,?,?,?,?,?,?,?,?)")) { - ps.setString(1, docId); - ps.setString(2, t(cd, "DocumentGenerated")); - ps.setString(3, t(cd, "Version")); - ps.setString(4, t(cd, "ContentDate")); - ps.setString(5, t(cd, "DataOperation")); - ps.setString(6, t(cd, "DataSupplier/SystemCountry")); - ps.setString(7, t(cd, "DataSupplier/Short")); - ps.setString(8, t(cd, "DataSupplier/Name")); - ps.setString(9, t(cd, "DataSupplier/Type")); - ps.executeUpdate(); - } - - NodeList funds = (NodeList) xp.evaluate("/FundsXML4/Funds/Fund", - doc, XPathConstants.NODESET); - for (int fi = 0; fi < funds.getLength(); fi++) { - Node fund = funds.item(fi); - int fundSeq = fi + 1; // 1-based document order - String ccy = t(fund, "Currency"); - Node tav = (Node) xp.evaluate( - "FundDynamicData/TotalAssetValues/TotalAssetValue", - fund, XPathConstants.NODE); - - try (PreparedStatement ps = c.prepareStatement( - "INSERT INTO fund VALUES (?,?,?,?,?,?,?,?)")) { - ps.setString(1, docId); - ps.setInt(2, fundSeq); - ps.setString(3, t(fund, "Identifiers/LEI")); - ps.setString(4, t(fund, "Names/OfficialName")); - ps.setString(5, ccy); - ps.setString(6, t(fund, "SingleFundFlag")); - ps.setString(7, t(tav, "NavDate")); - ps.setDouble(8, Double.parseDouble(t(tav, - "TotalNetAssetValue/Amount[@ccy='" + ccy + "']"))); - ps.executeUpdate(); - } - - NodeList scs = (NodeList) xp.evaluate( - "SingleFund/ShareClasses/ShareClass", fund, - XPathConstants.NODESET); - for (int si = 0; si < scs.getLength(); si++) { - Node sc = scs.item(si); - try (PreparedStatement ps = c.prepareStatement( - "INSERT INTO share_class VALUES (?,?,?,?,?,?,?,?)")) { - ps.setString(1, docId); - ps.setInt(2, fundSeq); - ps.setString(3, t(sc, "Identifiers/ISIN")); - ps.setString(4, t(sc, "Names/OfficialName")); - ps.setString(5, t(sc, "Currency")); - setNum(ps, 6, num(t(sc, "Prices/Price/NavPrice"))); - setNum(ps, 7, num(t(sc, "TotalAssetValues/" - + "TotalAssetValue/TotalNetAssetValue/Amount[@ccy='" - + ccy + "']"))); - setNum(ps, 8, num(t(sc, "TotalAssetValues/" - + "TotalAssetValue/SharesOutstanding"))); - ps.executeUpdate(); - } - } - - NodeList ports = (NodeList) xp.evaluate( - "FundDynamicData/Portfolios/Portfolio", fund, - XPathConstants.NODESET); - for (int pi = 0; pi < ports.getLength(); pi++) { - Node port = ports.item(pi); - int portSeq = pi + 1; - try (PreparedStatement ps = c.prepareStatement( - "INSERT INTO portfolio VALUES (?,?,?,?)")) { - ps.setString(1, docId); - ps.setInt(2, fundSeq); - ps.setInt(3, portSeq); - ps.setString(4, t(port, "NavDate")); - ps.executeUpdate(); - } - NodeList poss = (NodeList) xp.evaluate( - "Positions/Position", port, XPathConstants.NODESET); - for (int qi = 0; qi < poss.getLength(); qi++) { - Node pos = poss.item(qi); - String kind = null; - for (Node ch = pos.getFirstChild(); ch != null; - ch = ch.getNextSibling()) - if (ch.getNodeType() == Node.ELEMENT_NODE - && POSITION_KINDS.contains(ch.getNodeName())) { - kind = ch.getNodeName(); - break; - } - Double qty = (kind != null && QTY_ELEM.containsKey(kind)) - ? num(t(pos, kind + "/" + QTY_ELEM.get(kind))) - : null; - try (PreparedStatement ps = c.prepareStatement( - "INSERT INTO position VALUES (?,?,?,?,?,?,?,?,?,?,?)")) { - ps.setString(1, docId); - ps.setInt(2, fundSeq); - ps.setInt(3, portSeq); - ps.setInt(4, qi + 1); - ps.setString(5, t(pos, "UniqueID")); - ps.setString(6, t(pos, "Identifiers/ISIN")); - ps.setString(7, t(pos, "Currency")); - ps.setDouble(8, Double.parseDouble(t(pos, - "TotalValue/Amount[@ccy='" + ccy + "']"))); - ps.setDouble(9, Double.parseDouble( - t(pos, "TotalPercentage"))); - ps.setString(10, kind); - setNum(ps, 11, qty); - ps.executeUpdate(); - } - } - } - } - - NodeList assets = (NodeList) xp.evaluate( - "/FundsXML4/AssetMasterData/Asset", doc, XPathConstants.NODESET); - for (int ai = 0; ai < assets.getLength(); ai++) { - Node a = assets.item(ai); - try (PreparedStatement ps = c.prepareStatement( - "INSERT INTO asset VALUES (?,?,?,?,?,?,?)")) { - ps.setString(1, docId); - ps.setString(2, t(a, "UniqueID")); - ps.setString(3, t(a, "Identifiers/ISIN")); - ps.setString(4, t(a, "Name")); - ps.setString(5, t(a, "AssetType")); - ps.setString(6, t(a, "Currency")); - ps.setString(7, t(a, "Country")); - ps.executeUpdate(); - } - } - c.commit(); - } - return docId; - } - - static void setNum(PreparedStatement ps, int i, Double v) throws Exception { - if (v == null) ps.setNull(i, java.sql.Types.DECIMAL); - else ps.setDouble(i, v); - } - - // ---- export : rows -> FundsXML ---------------------------------------- - static void export(String db, String docId, String out) throws Exception { - Document doc = DocumentBuilderFactory.newInstance() - .newDocumentBuilder().newDocument(); - Element root = doc.createElement("FundsXML4"); - root.setAttribute("xmlns:xsi", XSI); - root.setAttribute("xsi:noNamespaceSchemaLocation", SCHEMA_URL); - doc.appendChild(root); - - try (Connection c = open(db)) { - ResultSet d = c.createStatement().executeQuery( - "SELECT * FROM document WHERE document_id='" + docId + "'"); - if (!d.next()) throw new RuntimeException("no document " + docId); - - Element cd = el(doc, root, "ControlData", null); - el(doc, cd, "UniqueDocumentID", d.getString("document_id")); - // Regenerate timestamp; xml_equiv.py ignores its value. - String gen = d.getString("generated"); - el(doc, cd, "DocumentGenerated", - gen != null ? gen : "2025-10-02T00:00:00"); - String ver = d.getString("version"); - if (ver != null) el(doc, cd, "Version", ver); // none for 4.0.0 - el(doc, cd, "ContentDate", d.getString("content_date")); - Element ds = el(doc, cd, "DataSupplier", null); - el(doc, ds, "SystemCountry", d.getString("supplier_country")); - el(doc, ds, "Short", d.getString("supplier_short")); - el(doc, ds, "Name", d.getString("supplier_name")); - el(doc, ds, "Type", d.getString("supplier_type")); - el(doc, cd, "DataOperation", d.getString("data_operation")); - - Element fundsEl = el(doc, root, "Funds", null); - PreparedStatement fps = c.prepareStatement( - "SELECT * FROM fund WHERE document_id=? ORDER BY fund_seq"); - fps.setString(1, docId); - ResultSet fr = fps.executeQuery(); - while (fr.next()) { - int fundSeq = fr.getInt("fund_seq"); - String ccy = fr.getString("currency"); - String navDate = fr.getString("nav_date"); - Element fund = el(doc, fundsEl, "Fund", null); - if (fr.getString("lei") != null) - el(doc, el(doc, fund, "Identifiers", null), "LEI", - fr.getString("lei")); - el(doc, el(doc, fund, "Names", null), "OfficialName", - fr.getString("official_name")); - el(doc, fund, "Currency", ccy); - if (fr.getString("single_fund_flag") != null) - el(doc, fund, "SingleFundFlag", - fr.getString("single_fund_flag")); - - Element fdd = el(doc, fund, "FundDynamicData", null); - Element tav = el(doc, el(doc, el(doc, fdd, - "TotalAssetValues", null), "TotalAssetValue", null), - "NavDate", navDate); - Element tavp = (Element) tav.getParentNode(); - el(doc, tavp, "TotalAssetNature", "OFFICIAL"); - Element tnav = el(doc, tavp, "TotalNetAssetValue", null); - Element amt = el(doc, tnav, "Amount", - f2(fr.getDouble("total_nav"))); - amt.setAttribute("ccy", ccy); - - Element ports = el(doc, fdd, "Portfolios", null); - PreparedStatement pps = c.prepareStatement( - "SELECT * FROM portfolio WHERE document_id=? AND fund_seq=?" - + " ORDER BY portfolio_seq"); - pps.setString(1, docId); pps.setInt(2, fundSeq); - ResultSet pr = pps.executeQuery(); - while (pr.next()) { - int portSeq = pr.getInt("portfolio_seq"); - Element pe = el(doc, ports, "Portfolio", null); - el(doc, pe, "NavDate", pr.getString("nav_date")); - Element poss = el(doc, pe, "Positions", null); - PreparedStatement qps = c.prepareStatement( - "SELECT * FROM position WHERE document_id=? AND " - + "fund_seq=? AND portfolio_seq=? ORDER BY position_seq"); - qps.setString(1, docId); qps.setInt(2, fundSeq); - qps.setInt(3, portSeq); - ResultSet qr = qps.executeQuery(); - while (qr.next()) { - Element pos = el(doc, poss, "Position", null); - el(doc, pos, "UniqueID", qr.getString("unique_id")); - if (qr.getString("isin") != null) - el(doc, el(doc, pos, "Identifiers", null), "ISIN", - qr.getString("isin")); - if (qr.getString("currency") != null) - el(doc, pos, "Currency", qr.getString("currency")); - Element tv = el(doc, el(doc, pos, "TotalValue", null), - "Amount", f2(qr.getDouble("value_fund_ccy"))); - tv.setAttribute("ccy", ccy); - el(doc, pos, "TotalPercentage", - f2(qr.getDouble("percentage"))); - String kind = qr.getString("kind"); - if (kind == null || !POSITION_KINDS.contains(kind)) - kind = "Generic"; - Element ke = el(doc, pos, kind, null); - Object q = qr.getObject("kind_qty"); - if (QTY_ELEM.containsKey(kind) && q != null) - el(doc, ke, QTY_ELEM.get(kind), - f2(((Number) q).doubleValue())); - } - } - - PreparedStatement sps = c.prepareStatement( - "SELECT * FROM share_class WHERE document_id=? AND " - + "fund_seq=? ORDER BY isin"); - sps.setString(1, docId); sps.setInt(2, fundSeq); - ResultSet sr = sps.executeQuery(); - Element sce = null; - while (sr.next()) { - if (sce == null) - sce = el(doc, el(doc, fund, "SingleFund", null), - "ShareClasses", null); - Element x = el(doc, sce, "ShareClass", null); - el(doc, el(doc, x, "Identifiers", null), "ISIN", - sr.getString("isin")); - if (sr.getString("official_name") != null) - el(doc, el(doc, x, "Names", null), "OfficialName", - sr.getString("official_name")); - el(doc, x, "Currency", sr.getString("currency")); - Object navp = sr.getObject("nav_price"); - if (navp != null) { - Element pe2 = el(doc, el(doc, x, "Prices", null), - "Price", null); - el(doc, pe2, "ActionCode", "C"); - el(doc, pe2, "NavDate", navDate); - el(doc, pe2, "PriceCurrency", sr.getString("currency")); - el(doc, pe2, "PriceNature", "OFFICIAL"); - el(doc, pe2, "NavPrice", - f2(((Number) navp).doubleValue())); - } - Object navf = sr.getObject("nav_fund_ccy"); - if (navf != null) { - Element t2 = el(doc, el(doc, x, "TotalAssetValues", - null), "TotalAssetValue", null); - el(doc, t2, "NavDate", navDate); - el(doc, t2, "TotalAssetNature", "OFFICIAL"); - Element a2 = el(doc, el(doc, t2, - "TotalNetAssetValue", null), "Amount", - f2(((Number) navf).doubleValue())); - a2.setAttribute("ccy", ccy); - Object so = sr.getObject("shares_outstanding"); - if (so != null) - el(doc, t2, "SharesOutstanding", String.format( - Locale.ROOT, "%.0f", - ((Number) so).doubleValue())); - } - } - } - - PreparedStatement aps = c.prepareStatement( - "SELECT * FROM asset WHERE document_id=? ORDER BY unique_id"); - aps.setString(1, docId); - ResultSet ar = aps.executeQuery(); - Element amd = null; - while (ar.next()) { - if (amd == null) amd = el(doc, root, "AssetMasterData", null); - Element ae = el(doc, amd, "Asset", null); - el(doc, ae, "UniqueID", ar.getString("unique_id")); - if (ar.getString("isin") != null) - el(doc, el(doc, ae, "Identifiers", null), "ISIN", - ar.getString("isin")); - el(doc, ae, "Currency", ar.getString("currency")); - if (ar.getString("country") != null) - el(doc, ae, "Country", ar.getString("country")); - el(doc, ae, "Name", ar.getString("name")); - el(doc, ae, "AssetType", ar.getString("asset_type")); - } - } - - Transformer tr = TransformerFactory.newInstance().newTransformer(); - tr.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); - tr.setOutputProperty(OutputKeys.INDENT, "yes"); - try (FileOutputStream os = new FileOutputStream(out)) { - tr.transform(new DOMSource(doc), new StreamResult(os)); - } - } - - // ---- cli --------------------------------------------------------------- - public static void main(String[] args) throws Exception { - if (args.length == 0) { System.err.println( - "usage: init|import|export|roundtrip ..."); System.exit(2); } - switch (args[0]) { - case "init" -> init(args[1]); - case "import" -> - System.out.println("imported " + doImport(args[1], args[2])); - case "export" -> { export(args[1], args[2], args[3]); - System.out.println("wrote " + args[3]); } - case "roundtrip" -> { - // import then export THROUGH the DB (the required test). - File tmp = File.createTempFile("fxdb", ".db"); - tmp.delete(); - init(tmp.getPath()); - String id = doImport(tmp.getPath(), args[1]); - export(tmp.getPath(), id, args[2]); - tmp.delete(); - System.out.println("round-trip ok: " + args[1] - + " -> DB -> " + args[2] + " (doc " + id + ")"); - } - default -> { System.err.println("unknown: " + args[0]); - System.exit(2); } - } - } -} diff --git a/Database_Integration/java/ImportFundsXml.java b/Database_Integration/java/ImportFundsXml.java new file mode 100644 index 0000000..a31fbf0 --- /dev/null +++ b/Database_Integration/java/ImportFundsXml.java @@ -0,0 +1,242 @@ +// ============================================================================= +// IMPORT — FundsXML file -> relational database (Java, SQLite via JDBC). +// +// Standalone, copy-me example of ONE direction (FundsXML -> DB). The reverse +// is a separate program, ExportFundsXml.java. Over-commented as documentation. +// +// DB SCHEMA ../ddl/schema.sql (document -> fund -> portfolio -> position; +// share_class per fund; asset document-scoped). Creates the schema in a +// fresh SQLite DB, then loads the file. +// +// RUN (sqlite-jdbc fetched by tools/fetch-tools.sh into .lib/) +// tools/fetch-tools.sh +// CP=.lib/sqlite-jdbc-3.46.1.3.jar +// javac -cp "$CP" -d /tmp/db Database_Integration/java/ImportFundsXml.java +// java --enable-native-access=ALL-UNNAMED -cp "$CP:/tmp/db" \ +// ImportFundsXml fx.db FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml +// +// DEPENDENCIES org.xerial:sqlite-jdbc + the JDK. XML via native javax.xml +// DOM/XPath — NO JAXB (a thin DOM binding is smaller and version-tolerant). +// +// FUNDSXML ASSUMPTIONS +// * No XML namespace -> bare element names in XPath. +// * Many //: all iterated; 1-based *_seq columns +// preserve order so the separate export reproduces the original document. +// * Positions link to AssetMasterData by shared -> `asset` is +// document-scoped. +// +// SECURITY the DOM parser forbids DOCTYPE/external entities (XXE). +// ============================================================================= +import java.io.FileInputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.Statement; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +public class ImportFundsXml { + + static final java.util.Set POSITION_KINDS = java.util.Set.of( + "Equity", "Bond", "ShareClass", "Warrant", "Certificate", "Option", + "Future", "FXForward", "Swap", "Repo", "RealEstate", "CallMoney", + "Account", "Generic"); + static final java.util.Map QTY_ELEM = java.util.Map.of( + "Equity", "Units", "Warrant", "Units", "Certificate", "Units", + "Bond", "Nominal", "ShareClass", "Shares", + "Option", "Contracts", "Future", "Contracts"); + + static XPath xpath() { return XPathFactory.newInstance().newXPath(); } + + /** First matching descendant's text (bare names; no namespace). */ + static String t(Node ctx, String expr) throws Exception { + Node n = (Node) xpath().evaluate(expr, ctx, XPathConstants.NODE); + return n == null ? null : n.getTextContent(); + } + static Double num(String s) { + return (s == null || s.isBlank()) ? null : Double.valueOf(s.trim()); + } + static void setNum(PreparedStatement ps, int i, Double v) throws Exception { + if (v == null) ps.setNull(i, java.sql.Types.DECIMAL); + else ps.setDouble(i, v); + } + + public static void main(String[] args) throws Exception { + if (args.length != 2) { + System.err.println("usage: ImportFundsXml "); + System.exit(2); + } + String db = args[0], xml = args[1]; + + // ---- parse the FundsXML file (hardened) --------------------------- + DocumentBuilderFactory f = DocumentBuilderFactory.newInstance(); + f.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + f.setFeature("http://xml.org/sax/features/external-general-entities", false); + f.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + f.setExpandEntityReferences(false); + Document doc; + try (FileInputStream in = new FileInputStream(xml)) { + doc = f.newDocumentBuilder().parse(in); + } + XPath xp = xpath(); + + try (Connection c = DriverManager.getConnection("jdbc:sqlite:" + db)) { + c.createStatement().execute("PRAGMA foreign_keys = ON"); + + // ---- create the schema (fresh DB) ----------------------------- + Path ddl = Path.of(System.getProperty("user.dir"), + "Database_Integration", "ddl", "schema.sql"); + if (!Files.exists(ddl)) ddl = Path.of("..", "ddl", "schema.sql"); + StringBuilder clean = new StringBuilder(); + for (String line : Files.readString(ddl).split("\n")) { + int cut = line.indexOf("--"); // strip line comments + clean.append(cut >= 0 ? line.substring(0, cut) : line) + .append('\n'); + } + try (Statement st = c.createStatement()) { + for (String stmt : clean.toString().split(";")) + if (!stmt.strip().isEmpty()) st.execute(stmt); + } + + c.setAutoCommit(false); + Node cd = (Node) xp.evaluate("/FundsXML4/ControlData", doc, + XPathConstants.NODE); + String docId = t(cd, "UniqueDocumentID"); + + try (PreparedStatement ps = c.prepareStatement( + "INSERT INTO document VALUES (?,?,?,?,?,?,?,?,?)")) { + ps.setString(1, docId); + ps.setString(2, t(cd, "DocumentGenerated")); + ps.setString(3, t(cd, "Version")); + ps.setString(4, t(cd, "ContentDate")); + ps.setString(5, t(cd, "DataOperation")); + ps.setString(6, t(cd, "DataSupplier/SystemCountry")); + ps.setString(7, t(cd, "DataSupplier/Short")); + ps.setString(8, t(cd, "DataSupplier/Name")); + ps.setString(9, t(cd, "DataSupplier/Type")); + ps.executeUpdate(); + } + + NodeList funds = (NodeList) xp.evaluate("/FundsXML4/Funds/Fund", + doc, XPathConstants.NODESET); + for (int fi = 0; fi < funds.getLength(); fi++) { + Node fund = funds.item(fi); + int fundSeq = fi + 1; // 1-based document order + String ccy = t(fund, "Currency"); + Node tav = (Node) xp.evaluate( + "FundDynamicData/TotalAssetValues/TotalAssetValue", + fund, XPathConstants.NODE); + try (PreparedStatement ps = c.prepareStatement( + "INSERT INTO fund VALUES (?,?,?,?,?,?,?,?)")) { + ps.setString(1, docId); + ps.setInt(2, fundSeq); + ps.setString(3, t(fund, "Identifiers/LEI")); + ps.setString(4, t(fund, "Names/OfficialName")); + ps.setString(5, ccy); + ps.setString(6, t(fund, "SingleFundFlag")); + ps.setString(7, t(tav, "NavDate")); + ps.setDouble(8, Double.parseDouble(t(tav, + "TotalNetAssetValue/Amount[@ccy='" + ccy + "']"))); + ps.executeUpdate(); + } + + NodeList scs = (NodeList) xp.evaluate( + "SingleFund/ShareClasses/ShareClass", fund, + XPathConstants.NODESET); + for (int si = 0; si < scs.getLength(); si++) { + Node sc = scs.item(si); + try (PreparedStatement ps = c.prepareStatement( + "INSERT INTO share_class VALUES (?,?,?,?,?,?,?,?)")) { + ps.setString(1, docId); + ps.setInt(2, fundSeq); + ps.setString(3, t(sc, "Identifiers/ISIN")); + ps.setString(4, t(sc, "Names/OfficialName")); + ps.setString(5, t(sc, "Currency")); + setNum(ps, 6, num(t(sc, "Prices/Price/NavPrice"))); + setNum(ps, 7, num(t(sc, "TotalAssetValues/" + + "TotalAssetValue/TotalNetAssetValue/Amount[@ccy='" + + ccy + "']"))); + setNum(ps, 8, num(t(sc, "TotalAssetValues/" + + "TotalAssetValue/SharesOutstanding"))); + ps.executeUpdate(); + } + } + + NodeList ports = (NodeList) xp.evaluate( + "FundDynamicData/Portfolios/Portfolio", fund, + XPathConstants.NODESET); + for (int pi = 0; pi < ports.getLength(); pi++) { + Node port = ports.item(pi); + int portSeq = pi + 1; + try (PreparedStatement ps = c.prepareStatement( + "INSERT INTO portfolio VALUES (?,?,?,?)")) { + ps.setString(1, docId); + ps.setInt(2, fundSeq); + ps.setInt(3, portSeq); + ps.setString(4, t(port, "NavDate")); + ps.executeUpdate(); + } + NodeList poss = (NodeList) xp.evaluate( + "Positions/Position", port, XPathConstants.NODESET); + for (int qi = 0; qi < poss.getLength(); qi++) { + Node pos = poss.item(qi); + String kind = null; + for (Node ch = pos.getFirstChild(); ch != null; + ch = ch.getNextSibling()) + if (ch.getNodeType() == Node.ELEMENT_NODE + && POSITION_KINDS.contains(ch.getNodeName())) { + kind = ch.getNodeName(); + break; + } + Double qty = (kind != null && QTY_ELEM.containsKey(kind)) + ? num(t(pos, kind + "/" + QTY_ELEM.get(kind))) + : null; + try (PreparedStatement ps = c.prepareStatement( + "INSERT INTO position VALUES (?,?,?,?,?,?,?,?,?,?,?)")) { + ps.setString(1, docId); + ps.setInt(2, fundSeq); + ps.setInt(3, portSeq); + ps.setInt(4, qi + 1); + ps.setString(5, t(pos, "UniqueID")); + ps.setString(6, t(pos, "Identifiers/ISIN")); + ps.setString(7, t(pos, "Currency")); + ps.setDouble(8, Double.parseDouble(t(pos, + "TotalValue/Amount[@ccy='" + ccy + "']"))); + ps.setDouble(9, Double.parseDouble( + t(pos, "TotalPercentage"))); + ps.setString(10, kind); + setNum(ps, 11, qty); + ps.executeUpdate(); + } + } + } + } + + NodeList assets = (NodeList) xp.evaluate( + "/FundsXML4/AssetMasterData/Asset", doc, XPathConstants.NODESET); + for (int ai = 0; ai < assets.getLength(); ai++) { + Node a = assets.item(ai); + try (PreparedStatement ps = c.prepareStatement( + "INSERT INTO asset VALUES (?,?,?,?,?,?,?)")) { + ps.setString(1, docId); + ps.setString(2, t(a, "UniqueID")); + ps.setString(3, t(a, "Identifiers/ISIN")); + ps.setString(4, t(a, "Name")); + ps.setString(5, t(a, "AssetType")); + ps.setString(6, t(a, "Currency")); + ps.setString(7, t(a, "Country")); + ps.executeUpdate(); + } + } + c.commit(); + System.out.println("imported document_id: " + docId); + } + } +} diff --git a/Database_Integration/javascript/export_fundsxml.mjs b/Database_Integration/javascript/export_fundsxml.mjs new file mode 100644 index 0000000..1e60e88 --- /dev/null +++ b/Database_Integration/javascript/export_fundsxml.mjs @@ -0,0 +1,181 @@ +// ============================================================================= +// EXPORT — relational database -> FundsXML file (Node.js, SQLite). +// +// Standalone, copy-me example of ONE direction (DB -> FundsXML). The reverse +// is a separate program, import_fundsxml.mjs. Over-commented as documentation. +// +// DB SCHEMA ../ddl/schema.sql (already populated by import_fundsxml.mjs). +// +// RUN +// node import_fundsxml.mjs fx.db some.xml +// node export_fundsxml.mjs fx.db FUNDSXML_MULTI_1 out.xml +// node ../tools/xml_equiv.py # (python) prove some.xml == out.xml +// +// DEPENDENCIES sql.js (WASM SQLite) + @xmldom/xmldom (DOM serialization). +// +// FUNDSXML NOTES +// * No XML namespace -> plain element names. +// * Output normalized to the 4.2.9 schema URL; constants the model does not +// store (TotalAssetNature=OFFICIAL, Price ActionCode=C / PriceNature= +// OFFICIAL) reproduced verbatim so the round-trip compares equal +// (../tools/xml_equiv.py, always paired with XSD validation). +// * ORDER BY the 1-based *_seq columns reproduces the original order of +// multiple funds / portfolios / positions. +// ============================================================================= +import { readFileSync, writeFileSync } from "node:fs"; +import initSqlJs from "sql.js"; +import { DOMParser, XMLSerializer } from "@xmldom/xmldom"; + +const SCHEMA_URL = + "https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd"; +const POSITION_KINDS = new Set(["Equity", "Bond", "ShareClass", "Warrant", + "Certificate", "Option", "Future", "FXForward", "Swap", "Repo", + "RealEstate", "CallMoney", "Account", "Generic"]); +const QTY_ELEM = { Equity: "Units", Warrant: "Units", Certificate: "Units", + Bond: "Nominal", ShareClass: "Shares", Option: "Contracts", + Future: "Contracts" }; +const f2 = (v) => Number(v).toFixed(2); + +function el(doc, parent, tag, text) { // the one XML-build primitive + const e = doc.createElement(tag); + if (text != null) e.appendChild(doc.createTextNode(String(text))); + parent.appendChild(e); + return e; +} +function rows(db, sql, params) { + const st = db.prepare(sql); + st.bind(params); + const out = []; + while (st.step()) out.push(st.getAsObject()); + st.free(); + return out; +} + +const [dbPath, docId, outPath] = process.argv.slice(2); +if (!dbPath || !docId || !outPath) { + console.error("usage: export_fundsxml.mjs "); + process.exit(2); +} + +const SQL = await initSqlJs(); +const db = new SQL.Database(readFileSync(dbPath)); + +const d = rows(db, "SELECT * FROM document WHERE document_id=?", [docId])[0]; +if (!d) throw new Error(`no document ${docId}`); + +const doc = new DOMParser().parseFromString("", "text/xml"); +const root = doc.documentElement; +root.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); +root.setAttribute("xsi:noNamespaceSchemaLocation", SCHEMA_URL); + +const cd = el(doc, root, "ControlData"); +el(doc, cd, "UniqueDocumentID", d.document_id); +// Regenerate the timestamp; xml_equiv.py ignores its value. +el(doc, cd, "DocumentGenerated", d.generated || "2025-10-02T00:00:00"); +if (d.version != null) el(doc, cd, "Version", d.version); // none for 4.0.0 +el(doc, cd, "ContentDate", d.content_date); +const ds = el(doc, cd, "DataSupplier"); +el(doc, ds, "SystemCountry", d.supplier_country); +el(doc, ds, "Short", d.supplier_short); +el(doc, ds, "Name", d.supplier_name); +el(doc, ds, "Type", d.supplier_type); +el(doc, cd, "DataOperation", d.data_operation); + +const fundsEl = el(doc, root, "Funds"); +for (const f of rows(db, + "SELECT * FROM fund WHERE document_id=? ORDER BY fund_seq", [docId])) { + const ccy = f.currency; + const fund = el(doc, fundsEl, "Fund"); + if (f.lei != null) el(doc, el(doc, fund, "Identifiers"), "LEI", f.lei); + el(doc, el(doc, fund, "Names"), "OfficialName", f.official_name); + el(doc, fund, "Currency", ccy); + if (f.single_fund_flag != null) + el(doc, fund, "SingleFundFlag", f.single_fund_flag); + + const fdd = el(doc, fund, "FundDynamicData"); + const tavp = el(doc, el(doc, fdd, "TotalAssetValues"), "TotalAssetValue"); + el(doc, tavp, "NavDate", f.nav_date); + el(doc, tavp, "TotalAssetNature", "OFFICIAL"); + const amt = el(doc, el(doc, tavp, "TotalNetAssetValue"), "Amount", + f2(f.total_nav)); + amt.setAttribute("ccy", ccy); + + const ports = el(doc, fdd, "Portfolios"); + for (const pf of rows(db, + "SELECT * FROM portfolio WHERE document_id=? AND fund_seq=? " + + "ORDER BY portfolio_seq", [docId, f.fund_seq])) { + const pe = el(doc, ports, "Portfolio"); + el(doc, pe, "NavDate", pf.nav_date); + const poss = el(doc, pe, "Positions"); + for (const p of rows(db, + "SELECT * FROM position WHERE document_id=? AND fund_seq=? AND " + + "portfolio_seq=? ORDER BY position_seq", + [docId, f.fund_seq, pf.portfolio_seq])) { + const pos = el(doc, poss, "Position"); + el(doc, pos, "UniqueID", p.unique_id); + if (p.isin != null) + el(doc, el(doc, pos, "Identifiers"), "ISIN", p.isin); + if (p.currency != null) el(doc, pos, "Currency", p.currency); + const tv = el(doc, el(doc, pos, "TotalValue"), "Amount", + f2(p.value_fund_ccy)); + tv.setAttribute("ccy", ccy); + el(doc, pos, "TotalPercentage", f2(p.percentage)); + const kind = POSITION_KINDS.has(p.kind) ? p.kind : "Generic"; + const ke = el(doc, pos, kind); + if (QTY_ELEM[kind] && p.kind_qty != null) + el(doc, ke, QTY_ELEM[kind], f2(p.kind_qty)); + } + } + + const scs = rows(db, "SELECT * FROM share_class WHERE document_id=? AND " + + "fund_seq=? ORDER BY isin", [docId, f.fund_seq]); + if (scs.length) { + const sce = el(doc, el(doc, fund, "SingleFund"), "ShareClasses"); + for (const sc of scs) { + const x = el(doc, sce, "ShareClass"); + el(doc, el(doc, x, "Identifiers"), "ISIN", sc.isin); + if (sc.official_name != null) + el(doc, el(doc, x, "Names"), "OfficialName", sc.official_name); + el(doc, x, "Currency", sc.currency); + if (sc.nav_price != null) { + const pr = el(doc, el(doc, x, "Prices"), "Price"); + el(doc, pr, "ActionCode", "C"); + el(doc, pr, "NavDate", f.nav_date); + el(doc, pr, "PriceCurrency", sc.currency); + el(doc, pr, "PriceNature", "OFFICIAL"); + el(doc, pr, "NavPrice", f2(sc.nav_price)); + } + if (sc.nav_fund_ccy != null) { + const t2 = el(doc, el(doc, x, "TotalAssetValues"), + "TotalAssetValue"); + el(doc, t2, "NavDate", f.nav_date); + el(doc, t2, "TotalAssetNature", "OFFICIAL"); + const a2 = el(doc, el(doc, t2, "TotalNetAssetValue"), "Amount", + f2(sc.nav_fund_ccy)); + a2.setAttribute("ccy", ccy); + if (sc.shares_outstanding != null) + el(doc, t2, "SharesOutstanding", + Number(sc.shares_outstanding).toFixed(0)); + } + } + } +} + +const assets = rows(db, + "SELECT * FROM asset WHERE document_id=? ORDER BY unique_id", [docId]); +if (assets.length) { + const amd = el(doc, root, "AssetMasterData"); + for (const a of assets) { + const ae = el(doc, amd, "Asset"); + el(doc, ae, "UniqueID", a.unique_id); + if (a.isin != null) el(doc, el(doc, ae, "Identifiers"), "ISIN", a.isin); + el(doc, ae, "Currency", a.currency); + if (a.country != null) el(doc, ae, "Country", a.country); + el(doc, ae, "Name", a.name); + el(doc, ae, "AssetType", a.asset_type); + } +} + +writeFileSync(outPath, '\n' + + new XMLSerializer().serializeToString(doc)); +console.log("wrote", outPath); diff --git a/Database_Integration/javascript/fundsxml_db.mjs b/Database_Integration/javascript/fundsxml_db.mjs deleted file mode 100644 index dbd81bb..0000000 --- a/Database_Integration/javascript/fundsxml_db.mjs +++ /dev/null @@ -1,317 +0,0 @@ -// ============================================================================= -// FundsXML <-> relational database — runnable reference (Node.js, SQLite). -// -// PURPOSE -// Standalone, copy-me example of BOTH directions (import a FundsXML file into -// the relational schema, export it back). Over-commented on purpose. -// -// DB SCHEMA ../ddl/schema.sql (document -> fund -> portfolio -> position; -// share_class per fund; asset document-scoped). -// -// RUN -// cd Database_Integration/javascript && npm install -// node fundsxml_db.mjs roundtrip \ -// ../../FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml out.xml -// commands: init | import | export | -// roundtrip -// -// DEPENDENCIES (all pure-JS, no native build — runs on any Node >= 18): -// sql.js SQLite compiled to WebAssembly (the DB engine) -// @xmldom/xmldom a namespace-free DOM (FundsXML 4.x has no namespace) -// xpath XPath 1.0 over that DOM -// Node 20 has no built-in node:sqlite (that arrived in 22.5), hence sql.js. -// -// FUNDSXML ASSUMPTIONS -// * No XML namespace -> bare element names in XPath. -// * Many //: all iterated; 1-based *_seq columns -// preserve order so the round-trip compares equal (../tools/xml_equiv.py). -// * Positions link to AssetMasterData by shared -> `asset` is -// document-scoped. -// * Export normalized to the 4.2.9 schema URL; constants the model does not -// store (TotalAssetNature=OFFICIAL, Price ActionCode=C / PriceNature= -// OFFICIAL) are reproduced verbatim. -// -// SECURITY -// @xmldom/xmldom does not resolve external entities/DTDs (no XXE surface). -// ============================================================================= -import { readFileSync, writeFileSync } from "node:fs"; -import { dirname, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import initSqlJs from "sql.js"; -import { DOMParser, XMLSerializer } from "@xmldom/xmldom"; -import xpath from "xpath"; - -const HERE = dirname(fileURLToPath(import.meta.url)); -const DDL = resolve(HERE, "..", "ddl", "schema.sql"); -const SCHEMA_URL = - "https://github.com/fundsxml/schema/releases/download/4.2.9/FundsXML.xsd"; - -// Position instrument-class element + its mandatory quantity child. Kinds not -// in QTY_ELEM are schema-valid as an empty element. -const POSITION_KINDS = new Set(["Equity", "Bond", "ShareClass", "Warrant", - "Certificate", "Option", "Future", "FXForward", "Swap", "Repo", - "RealEstate", "CallMoney", "Account", "Generic"]); -const QTY_ELEM = { Equity: "Units", Warrant: "Units", Certificate: "Units", - Bond: "Nominal", ShareClass: "Shares", Option: "Contracts", - Future: "Contracts" }; - -// ---- tiny helpers ---------------------------------------------------------- -const txt = (ctx, expr) => { // first match's text or null - const n = xpath.select1(expr, ctx); - return n && n.textContent != null && n.textContent !== "" ? n.textContent : null; -}; -const numOrNull = (s) => (s == null || s === "" ? null : Number(s)); -const f2 = (v) => Number(v).toFixed(2); - -function el(doc, parent, tag, text) { // the one XML-build primitive - const e = doc.createElement(tag); - if (text != null) e.appendChild(doc.createTextNode(String(text))); - parent.appendChild(e); - return e; -} - -function execScript(db, sql) { - // Strip "--" line comments (no string literals in the DDL) then run. - const clean = sql.split("\n").map((l) => { - const i = l.indexOf("--"); - return i >= 0 ? l.slice(0, i) : l; - }).join("\n"); - for (const stmt of clean.split(";")) if (stmt.trim()) db.run(stmt); -} - -function parseSecure(xmlPath) { - // @xmldom/xmldom never expands external entities; we additionally reject a - // DOCTYPE so a malicious feed cannot even declare entities. - const src = readFileSync(xmlPath, "utf8"); - if (/ rows -------------------------------------------- -function doImport(db, xmlPath) { - const doc = parseSecure(xmlPath); - const cd = xpath.select1("/FundsXML4/ControlData", doc); - const docId = txt(cd, "UniqueDocumentID"); - - db.run("INSERT INTO document VALUES (?,?,?,?,?,?,?,?,?)", [ - docId, txt(cd, "DocumentGenerated"), txt(cd, "Version"), - txt(cd, "ContentDate"), txt(cd, "DataOperation"), - txt(cd, "DataSupplier/SystemCountry"), txt(cd, "DataSupplier/Short"), - txt(cd, "DataSupplier/Name"), txt(cd, "DataSupplier/Type")]); - - const funds = xpath.select("/FundsXML4/Funds/Fund", doc); - funds.forEach((fund, fi) => { - const fundSeq = fi + 1; // 1-based document order - const ccy = txt(fund, "Currency"); - const tav = xpath.select1( - "FundDynamicData/TotalAssetValues/TotalAssetValue", fund); - db.run("INSERT INTO fund VALUES (?,?,?,?,?,?,?,?)", [ - docId, fundSeq, txt(fund, "Identifiers/LEI"), - txt(fund, "Names/OfficialName"), ccy, txt(fund, "SingleFundFlag"), - txt(tav, "NavDate"), - Number(txt(tav, `TotalNetAssetValue/Amount[@ccy='${ccy}']`))]); - - for (const sc of xpath.select("SingleFund/ShareClasses/ShareClass", fund)) { - db.run("INSERT INTO share_class VALUES (?,?,?,?,?,?,?,?)", [ - docId, fundSeq, txt(sc, "Identifiers/ISIN"), - txt(sc, "Names/OfficialName"), txt(sc, "Currency"), - numOrNull(txt(sc, "Prices/Price/NavPrice")), - numOrNull(txt(sc, - `TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount[@ccy='${ccy}']`)), - numOrNull(txt(sc, - "TotalAssetValues/TotalAssetValue/SharesOutstanding"))]); - } - - xpath.select("FundDynamicData/Portfolios/Portfolio", fund) - .forEach((port, pi) => { - const portSeq = pi + 1; - db.run("INSERT INTO portfolio VALUES (?,?,?,?)", - [docId, fundSeq, portSeq, txt(port, "NavDate")]); - xpath.select("Positions/Position", port).forEach((pos, qi) => { - let kind = null; - for (let ch = pos.firstChild; ch; ch = ch.nextSibling) - if (ch.nodeType === 1 && POSITION_KINDS.has(ch.nodeName)) { - kind = ch.nodeName; break; - } - const qty = kind && QTY_ELEM[kind] - ? numOrNull(txt(pos, `${kind}/${QTY_ELEM[kind]}`)) : null; - db.run("INSERT INTO position VALUES (?,?,?,?,?,?,?,?,?,?,?)", [ - docId, fundSeq, portSeq, qi + 1, txt(pos, "UniqueID"), - txt(pos, "Identifiers/ISIN"), txt(pos, "Currency"), - Number(txt(pos, `TotalValue/Amount[@ccy='${ccy}']`)), - Number(txt(pos, "TotalPercentage")), kind, qty]); - }); - }); - }); - - for (const a of xpath.select("/FundsXML4/AssetMasterData/Asset", doc)) { - db.run("INSERT INTO asset VALUES (?,?,?,?,?,?,?)", [ - docId, txt(a, "UniqueID"), txt(a, "Identifiers/ISIN"), - txt(a, "Name"), txt(a, "AssetType"), txt(a, "Currency"), - txt(a, "Country")]); - } - return docId; -} - -// ---- export : rows -> FundsXML -------------------------------------------- -function rows(db, sql, params) { - const st = db.prepare(sql); - st.bind(params); - const out = []; - while (st.step()) out.push(st.getAsObject()); - st.free(); - return out; -} - -function exportXml(db, docId, outPath) { - const d = rows(db, "SELECT * FROM document WHERE document_id=?", [docId])[0]; - if (!d) throw new Error(`no document ${docId}`); - - const doc = new DOMParser().parseFromString("", "text/xml"); - const root = doc.documentElement; - root.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); - root.setAttribute("xsi:noNamespaceSchemaLocation", SCHEMA_URL); - - const cd = el(doc, root, "ControlData"); - el(doc, cd, "UniqueDocumentID", d.document_id); - // Regenerate the timestamp; xml_equiv.py ignores its value. - el(doc, cd, "DocumentGenerated", d.generated || "2025-10-02T00:00:00"); - if (d.version != null) el(doc, cd, "Version", d.version); // none for 4.0.0 - el(doc, cd, "ContentDate", d.content_date); - const ds = el(doc, cd, "DataSupplier"); - el(doc, ds, "SystemCountry", d.supplier_country); - el(doc, ds, "Short", d.supplier_short); - el(doc, ds, "Name", d.supplier_name); - el(doc, ds, "Type", d.supplier_type); - el(doc, cd, "DataOperation", d.data_operation); - - const fundsEl = el(doc, root, "Funds"); - for (const f of rows(db, - "SELECT * FROM fund WHERE document_id=? ORDER BY fund_seq", [docId])) { - const ccy = f.currency; - const fund = el(doc, fundsEl, "Fund"); - if (f.lei != null) el(doc, el(doc, fund, "Identifiers"), "LEI", f.lei); - el(doc, el(doc, fund, "Names"), "OfficialName", f.official_name); - el(doc, fund, "Currency", ccy); - if (f.single_fund_flag != null) - el(doc, fund, "SingleFundFlag", f.single_fund_flag); - - const fdd = el(doc, fund, "FundDynamicData"); - const tavp = el(doc, el(doc, fdd, "TotalAssetValues"), "TotalAssetValue"); - el(doc, tavp, "NavDate", f.nav_date); - el(doc, tavp, "TotalAssetNature", "OFFICIAL"); - const amt = el(doc, el(doc, tavp, "TotalNetAssetValue"), "Amount", - f2(f.total_nav)); - amt.setAttribute("ccy", ccy); - - const ports = el(doc, fdd, "Portfolios"); - for (const pf of rows(db, - "SELECT * FROM portfolio WHERE document_id=? AND fund_seq=? " - + "ORDER BY portfolio_seq", [docId, f.fund_seq])) { - const pe = el(doc, ports, "Portfolio"); - el(doc, pe, "NavDate", pf.nav_date); - const poss = el(doc, pe, "Positions"); - for (const p of rows(db, - "SELECT * FROM position WHERE document_id=? AND fund_seq=? AND " - + "portfolio_seq=? ORDER BY position_seq", - [docId, f.fund_seq, pf.portfolio_seq])) { - const pos = el(doc, poss, "Position"); - el(doc, pos, "UniqueID", p.unique_id); - if (p.isin != null) - el(doc, el(doc, pos, "Identifiers"), "ISIN", p.isin); - if (p.currency != null) el(doc, pos, "Currency", p.currency); - const tv = el(doc, el(doc, pos, "TotalValue"), "Amount", - f2(p.value_fund_ccy)); - tv.setAttribute("ccy", ccy); - el(doc, pos, "TotalPercentage", f2(p.percentage)); - const kind = POSITION_KINDS.has(p.kind) ? p.kind : "Generic"; - const ke = el(doc, pos, kind); - if (QTY_ELEM[kind] && p.kind_qty != null) - el(doc, ke, QTY_ELEM[kind], f2(p.kind_qty)); - } - } - - const scs = rows(db, "SELECT * FROM share_class WHERE document_id=? AND " - + "fund_seq=? ORDER BY isin", [docId, f.fund_seq]); - if (scs.length) { - const sce = el(doc, el(doc, fund, "SingleFund"), "ShareClasses"); - for (const sc of scs) { - const x = el(doc, sce, "ShareClass"); - el(doc, el(doc, x, "Identifiers"), "ISIN", sc.isin); - if (sc.official_name != null) - el(doc, el(doc, x, "Names"), "OfficialName", sc.official_name); - el(doc, x, "Currency", sc.currency); - if (sc.nav_price != null) { - const pr = el(doc, el(doc, x, "Prices"), "Price"); - el(doc, pr, "ActionCode", "C"); - el(doc, pr, "NavDate", f.nav_date); - el(doc, pr, "PriceCurrency", sc.currency); - el(doc, pr, "PriceNature", "OFFICIAL"); - el(doc, pr, "NavPrice", f2(sc.nav_price)); - } - if (sc.nav_fund_ccy != null) { - const t2 = el(doc, el(doc, x, "TotalAssetValues"), - "TotalAssetValue"); - el(doc, t2, "NavDate", f.nav_date); - el(doc, t2, "TotalAssetNature", "OFFICIAL"); - const a2 = el(doc, el(doc, t2, "TotalNetAssetValue"), "Amount", - f2(sc.nav_fund_ccy)); - a2.setAttribute("ccy", ccy); - if (sc.shares_outstanding != null) - el(doc, t2, "SharesOutstanding", - Number(sc.shares_outstanding).toFixed(0)); - } - } - } - } - - const assets = rows(db, - "SELECT * FROM asset WHERE document_id=? ORDER BY unique_id", [docId]); - if (assets.length) { - const amd = el(doc, root, "AssetMasterData"); - for (const a of assets) { - const ae = el(doc, amd, "Asset"); - el(doc, ae, "UniqueID", a.unique_id); - if (a.isin != null) - el(doc, el(doc, ae, "Identifiers"), "ISIN", a.isin); - el(doc, ae, "Currency", a.currency); - if (a.country != null) el(doc, ae, "Country", a.country); - el(doc, ae, "Name", a.name); - el(doc, ae, "AssetType", a.asset_type); - } - } - - const xml = '\n' - + new XMLSerializer().serializeToString(doc); - writeFileSync(outPath, xml); -} - -// ---- cli ------------------------------------------------------------------- -const [cmd, a1, a2, a3] = process.argv.slice(2); -const SQL = await initSqlJs(); -if (cmd === "init") { - const db = new SQL.Database(); - execScript(db, readFileSync(DDL, "utf8")); - persist(db, a1); -} else if (cmd === "import") { - const db = load(SQL, a1); - console.log("imported", doImport(db, a2)); - persist(db, a1); -} else if (cmd === "export") { - exportXml(load(SQL, a1), a2, a3); - console.log("wrote", a3); -} else if (cmd === "roundtrip") { - // import then export THROUGH an (in-memory) DB — the required test. - const db = new SQL.Database(); - execScript(db, readFileSync(DDL, "utf8")); - const id = doImport(db, a1); - exportXml(db, id, a2); - console.log(`round-trip ok: ${a1} -> DB -> ${a2} (doc ${id})`); -} else { - console.error("usage: init|import|export|roundtrip ..."); - process.exit(2); -} diff --git a/Database_Integration/javascript/import_fundsxml.mjs b/Database_Integration/javascript/import_fundsxml.mjs new file mode 100644 index 0000000..176bda1 --- /dev/null +++ b/Database_Integration/javascript/import_fundsxml.mjs @@ -0,0 +1,139 @@ +// ============================================================================= +// IMPORT — FundsXML file -> relational database (Node.js, SQLite). +// +// Standalone, copy-me example of ONE direction (FundsXML -> DB). The reverse +// is a separate program, export_fundsxml.mjs. Over-commented as documentation. +// +// DB SCHEMA ../ddl/schema.sql (document -> fund -> portfolio -> position; +// share_class per fund; asset document-scoped). Creates the schema in a +// fresh DB then loads the file. The "db file" is sql.js's serialized image. +// +// RUN +// cd Database_Integration/javascript && npm install +// node import_fundsxml.mjs fx.db \ +// ../../FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml +// node export_fundsxml.mjs fx.db FUNDSXML_MULTI_1 out.xml +// +// DEPENDENCIES (pure-JS, no native build): sql.js (WASM SQLite), +// @xmldom/xmldom + xpath (namespace-free DOM). Node 20 has no built-in +// node:sqlite (added in 22.5), hence sql.js. +// +// FUNDSXML ASSUMPTIONS +// * No XML namespace -> bare element names in XPath. +// * Many //: all iterated; 1-based *_seq columns +// preserve order so the separate export reproduces the original document. +// * Positions link to AssetMasterData by shared -> `asset` is +// document-scoped. +// +// SECURITY @xmldom/xmldom never resolves external entities; we also reject a +// DOCTYPE so a feed cannot declare entities (XXE). +// ============================================================================= +import { readFileSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import initSqlJs from "sql.js"; +import { DOMParser } from "@xmldom/xmldom"; +import xpath from "xpath"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const DDL = resolve(HERE, "..", "ddl", "schema.sql"); + +const POSITION_KINDS = new Set(["Equity", "Bond", "ShareClass", "Warrant", + "Certificate", "Option", "Future", "FXForward", "Swap", "Repo", + "RealEstate", "CallMoney", "Account", "Generic"]); +const QTY_ELEM = { Equity: "Units", Warrant: "Units", Certificate: "Units", + Bond: "Nominal", ShareClass: "Shares", Option: "Contracts", + Future: "Contracts" }; + +const txt = (ctx, expr) => { // first match's text or null + const n = xpath.select1(expr, ctx); + return n && n.textContent !== "" && n.textContent != null + ? n.textContent : null; +}; +const numOrNull = (s) => (s == null || s === "" ? null : Number(s)); + +function execScript(db, sql) { + // Strip "--" line comments (no string literals in the DDL) then run. + const clean = sql.split("\n").map((l) => { + const i = l.indexOf("--"); + return i >= 0 ? l.slice(0, i) : l; + }).join("\n"); + for (const stmt of clean.split(";")) if (stmt.trim()) db.run(stmt); +} + +const [dbPath, xmlPath] = process.argv.slice(2); +if (!dbPath || !xmlPath) { + console.error("usage: import_fundsxml.mjs "); + process.exit(2); +} + +const src = readFileSync(xmlPath, "utf8"); +if (/ { + const fundSeq = fi + 1; // 1-based document order + const ccy = txt(fund, "Currency"); + const tav = xpath.select1( + "FundDynamicData/TotalAssetValues/TotalAssetValue", fund); + db.run("INSERT INTO fund VALUES (?,?,?,?,?,?,?,?)", [ + docId, fundSeq, txt(fund, "Identifiers/LEI"), + txt(fund, "Names/OfficialName"), ccy, txt(fund, "SingleFundFlag"), + txt(tav, "NavDate"), + Number(txt(tav, `TotalNetAssetValue/Amount[@ccy='${ccy}']`))]); + + for (const sc of xpath.select("SingleFund/ShareClasses/ShareClass", fund)) { + db.run("INSERT INTO share_class VALUES (?,?,?,?,?,?,?,?)", [ + docId, fundSeq, txt(sc, "Identifiers/ISIN"), + txt(sc, "Names/OfficialName"), txt(sc, "Currency"), + numOrNull(txt(sc, "Prices/Price/NavPrice")), + numOrNull(txt(sc, + `TotalAssetValues/TotalAssetValue/TotalNetAssetValue/Amount[@ccy='${ccy}']`)), + numOrNull(txt(sc, + "TotalAssetValues/TotalAssetValue/SharesOutstanding"))]); + } + + xpath.select("FundDynamicData/Portfolios/Portfolio", fund) + .forEach((port, pi) => { + const portSeq = pi + 1; + db.run("INSERT INTO portfolio VALUES (?,?,?,?)", + [docId, fundSeq, portSeq, txt(port, "NavDate")]); + xpath.select("Positions/Position", port).forEach((pos, qi) => { + let kind = null; + for (let ch = pos.firstChild; ch; ch = ch.nextSibling) + if (ch.nodeType === 1 && POSITION_KINDS.has(ch.nodeName)) { + kind = ch.nodeName; break; + } + const qty = kind && QTY_ELEM[kind] + ? numOrNull(txt(pos, `${kind}/${QTY_ELEM[kind]}`)) : null; + db.run("INSERT INTO position VALUES (?,?,?,?,?,?,?,?,?,?,?)", [ + docId, fundSeq, portSeq, qi + 1, txt(pos, "UniqueID"), + txt(pos, "Identifiers/ISIN"), txt(pos, "Currency"), + Number(txt(pos, `TotalValue/Amount[@ccy='${ccy}']`)), + Number(txt(pos, "TotalPercentage")), kind, qty]); + }); + }); +}); + +for (const a of xpath.select("/FundsXML4/AssetMasterData/Asset", doc)) { + db.run("INSERT INTO asset VALUES (?,?,?,?,?,?,?)", [ + docId, txt(a, "UniqueID"), txt(a, "Identifiers/ISIN"), + txt(a, "Name"), txt(a, "AssetType"), txt(a, "Currency"), + txt(a, "Country")]); +} + +writeFileSync(dbPath, Buffer.from(db.export())); +console.log("imported document_id:", docId); diff --git a/Database_Integration/python/export_fundsxml.py b/Database_Integration/python/export_fundsxml.py new file mode 100644 index 0000000..ae17685 --- /dev/null +++ b/Database_Integration/python/export_fundsxml.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +# ============================================================================= +# EXPORT — relational database -> FundsXML file (Python 3, SQLite). +# +# Standalone, copy-me example of ONE direction only: turning the relational +# rows back into a FundsXML document. The reverse (FundsXML -> DB) is a +# separate program, import_fundsxml.py. Over-commented as documentation. +# +# DB SCHEMA ../ddl/schema.sql (already populated by import_fundsxml.py). +# +# RUN +# python3 import_fundsxml.py fx.db some.xml # produces a doc id +# python3 export_fundsxml.py fx.db FUNDSXML_MULTI_1 out.xml +# +# Prove the round-trip (import file vs exported file): +# python3 ../tools/xml_equiv.py some.xml out.xml +# xmllint --noout --schema .schema-cache/4.2.9/FundsXML.xsd out.xml +# +# DEPENDENCIES Python stdlib `sqlite3` + `lxml`. +# +# FUNDSXML NOTES +# * No XML namespace -> plain element names. +# * Output is normalized to the 4.2.9 schema URL. Constants the model does +# not store (TotalAssetNature=OFFICIAL, Price ActionCode=C / PriceNature= +# OFFICIAL) are reproduced verbatim so the round-trip compares equal +# (xml_equiv.py ignores the DocumentGenerated timestamp & number/whitespace +# formatting; it is always paired with XSD validation). +# * Rows are emitted ordered by the 1-based *_seq columns the import wrote, +# so multiple funds/portfolios/positions come back in the original order. +# ============================================================================= +import sqlite3 +import sys +from datetime import datetime, timezone + +from lxml import etree + +SCHEMA_URL = ("https://github.com/fundsxml/schema/releases/download/" + "4.2.9/FundsXML.xsd") +XSI = "http://www.w3.org/2001/XMLSchema-instance" +POSITION_KINDS = {"Equity", "Bond", "ShareClass", "Warrant", "Certificate", + "Option", "Future", "FXForward", "Swap", "Repo", + "RealEstate", "CallMoney", "Account", "Generic"} +QTY_ELEM = {"Equity": "Units", "Warrant": "Units", "Certificate": "Units", + "Bond": "Nominal", "ShareClass": "Shares", + "Option": "Contracts", "Future": "Contracts"} + + +def _el(parent, tag, text=None, **attrs): + """Append a child element — the single XML-building primitive.""" + e = etree.SubElement(parent, tag) + for k, v in attrs.items(): + e.set(k, str(v)) + if text is not None: + e.text = str(text) + return e + + +def main() -> int: + if len(sys.argv) != 4: + print("usage: export_fundsxml.py ", + file=sys.stderr) + return 2 + db, document_id, out_path = sys.argv[1], sys.argv[2], sys.argv[3] + + con = sqlite3.connect(db) + con.row_factory = sqlite3.Row + d = con.execute("SELECT * FROM document WHERE document_id=?", + (document_id,)).fetchone() + if d is None: + raise SystemExit(f"no document {document_id!r} in {db}") + + root = etree.Element("FundsXML4", nsmap={"xsi": XSI}) + root.set(f"{{{XSI}}}noNamespaceSchemaLocation", SCHEMA_URL) + + cd = _el(root, "ControlData") + _el(cd, "UniqueDocumentID", d["document_id"]) + # Regenerate the timestamp; xml_equiv.py ignores its value by design. + _el(cd, "DocumentGenerated", + d["generated"] or datetime.now(timezone.utc) + .strftime("%Y-%m-%dT%H:%M:%S")) + if d["version"]: # absent for FundsXML 4.0.0 + _el(cd, "Version", d["version"]) + _el(cd, "ContentDate", d["content_date"]) + ds = _el(cd, "DataSupplier") + _el(ds, "SystemCountry", d["supplier_country"]) + _el(ds, "Short", d["supplier_short"]) + _el(ds, "Name", d["supplier_name"]) + _el(ds, "Type", d["supplier_type"]) + _el(cd, "DataOperation", d["data_operation"]) + + funds_el = _el(root, "Funds") + for f in con.execute( + "SELECT * FROM fund WHERE document_id=? ORDER BY fund_seq", + (document_id,)): + ccy = f["currency"] + fund = _el(funds_el, "Fund") + if f["lei"]: + _el(_el(fund, "Identifiers"), "LEI", f["lei"]) + _el(_el(fund, "Names"), "OfficialName", f["official_name"]) + _el(fund, "Currency", ccy) + if f["single_fund_flag"]: + _el(fund, "SingleFundFlag", f["single_fund_flag"]) + + fdd = _el(fund, "FundDynamicData") + tav = _el(_el(_el(fdd, "TotalAssetValues"), "TotalAssetValue"), + "NavDate", f["nav_date"]).getparent() + _el(tav, "TotalAssetNature", "OFFICIAL") + _el(_el(tav, "TotalNetAssetValue"), "Amount", + f'{f["total_nav"]:.2f}', ccy=ccy) + + ports = _el(fdd, "Portfolios") + for pf in con.execute( + "SELECT * FROM portfolio WHERE document_id=? AND fund_seq=? " + "ORDER BY portfolio_seq", (document_id, f["fund_seq"])): + pe = _el(ports, "Portfolio") + _el(pe, "NavDate", pf["nav_date"]) + poss = _el(pe, "Positions") + for p in con.execute( + "SELECT * FROM position WHERE document_id=? AND fund_seq=? " + "AND portfolio_seq=? ORDER BY position_seq", + (document_id, f["fund_seq"], pf["portfolio_seq"])): + pos = _el(poss, "Position") + _el(pos, "UniqueID", p["unique_id"]) + if p["isin"]: + _el(_el(pos, "Identifiers"), "ISIN", p["isin"]) + if p["currency"]: + _el(pos, "Currency", p["currency"]) + _el(_el(pos, "TotalValue"), "Amount", + f'{p["value_fund_ccy"]:.2f}', ccy=ccy) + _el(pos, "TotalPercentage", f'{p["percentage"]:.2f}') + kind = p["kind"] if p["kind"] in POSITION_KINDS else "Generic" + ke = _el(pos, kind) + if kind in QTY_ELEM and p["kind_qty"] is not None: + _el(ke, QTY_ELEM[kind], f'{p["kind_qty"]:.2f}') + + scs = con.execute( + "SELECT * FROM share_class WHERE document_id=? AND fund_seq=? " + "ORDER BY isin", (document_id, f["fund_seq"])).fetchall() + if scs: + sce = _el(_el(fund, "SingleFund"), "ShareClasses") + for sc in scs: + x = _el(sce, "ShareClass") + _el(_el(x, "Identifiers"), "ISIN", sc["isin"]) + if sc["official_name"]: + _el(_el(x, "Names"), "OfficialName", sc["official_name"]) + _el(x, "Currency", sc["currency"]) + if sc["nav_price"] is not None: + pr = _el(_el(x, "Prices"), "Price") + _el(pr, "ActionCode", "C") + _el(pr, "NavDate", f["nav_date"]) + _el(pr, "PriceCurrency", sc["currency"]) + _el(pr, "PriceNature", "OFFICIAL") + _el(pr, "NavPrice", f'{sc["nav_price"]:.2f}') + if sc["nav_fund_ccy"] is not None: + t = _el(_el(x, "TotalAssetValues"), "TotalAssetValue") + _el(t, "NavDate", f["nav_date"]) + _el(t, "TotalAssetNature", "OFFICIAL") + _el(_el(t, "TotalNetAssetValue"), "Amount", + f'{sc["nav_fund_ccy"]:.2f}', ccy=ccy) + if sc["shares_outstanding"] is not None: + _el(t, "SharesOutstanding", + f'{sc["shares_outstanding"]:.0f}') + + assets = con.execute( + "SELECT * FROM asset WHERE document_id=? ORDER BY unique_id", + (document_id,)).fetchall() + if assets: + amd = _el(root, "AssetMasterData") + for a in assets: + ae = _el(amd, "Asset") + _el(ae, "UniqueID", a["unique_id"]) + if a["isin"]: + _el(_el(ae, "Identifiers"), "ISIN", a["isin"]) + _el(ae, "Currency", a["currency"]) + if a["country"]: + _el(ae, "Country", a["country"]) + _el(ae, "Name", a["name"]) + _el(ae, "AssetType", a["asset_type"]) + con.close() + + etree.ElementTree(root).write(out_path, xml_declaration=True, + encoding="UTF-8", pretty_print=True) + print("wrote", out_path) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Database_Integration/python/fundsxml_db.py b/Database_Integration/python/fundsxml_db.py deleted file mode 100644 index 6f0bb49..0000000 --- a/Database_Integration/python/fundsxml_db.py +++ /dev/null @@ -1,362 +0,0 @@ -#!/usr/bin/env python3 -# ============================================================================= -# FundsXML <-> relational database — runnable reference (Python 3, SQLite). -# -# PURPOSE -# A standalone, copy-me example showing BOTH directions: -# * import : read a FundsXML file -> rows in the relational schema -# * export : read those rows -> a FundsXML file -# It is deliberately over-commented: read it top to bottom and you can -# reimplement the same pattern in any language/RDBMS. -# -# DB SCHEMA -# ../ddl/schema.sql (document -> fund -> portfolio -> position; share_class -# per fund; asset document-scoped). SQLite is used so the example runs with -# zero setup; the SQL is plain enough to move to Postgres/Oracle/SQL Server. -# -# RUN -# python3 fundsxml_db.py init fx.db -# python3 fundsxml_db.py import fx.db ../../FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml -# python3 fundsxml_db.py export fx.db FUNDSXML_MULTI_1 out.xml -# python3 fundsxml_db.py roundtrip ../../FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml out.xml -# -# DEPENDENCIES -# Python stdlib `sqlite3` + `lxml` (already used elsewhere in this repo). -# -# FUNDSXML ASSUMPTIONS (important and non-obvious) -# * FundsXML 4.x has NO XML namespace -> XPath uses bare element names. -# * A document may contain MANY , each MANY , each MANY -# . We iterate all of them (no [0] shortcuts) and remember their -# order with 1-based *_seq columns so the export is byte-faithful. -# * Positions link to AssetMasterData by a shared (FundsXML's -# own join key), so `asset` is document-scoped, not per-fund. -# * The export is normalized to the 4.2.9 schema URL; round-trip equality is -# checked with ../tools/xml_equiv.py, which ignores the volatile -# ControlData/DocumentGenerated timestamp and numeric/whitespace -# formatting (see that file for the exact rules). -# -# SECURITY -# The XML parser disables network access and entity resolution (XXE / billion -# laughs). FundsXML never needs DTDs or external entities. -# ============================================================================= -import sqlite3 -import sys -import tempfile -from datetime import datetime, timezone -from pathlib import Path - -from lxml import etree - -DDL = Path(__file__).resolve().parents[1] / "ddl" / "schema.sql" - -# The official released schema for the version we normalize exports to. Set as -# xsi:noNamespaceSchemaLocation so any validator can find the XSD. -SCHEMA_URL = ("https://github.com/fundsxml/schema/releases/download/" - "4.2.9/FundsXML.xsd") -XSI = "http://www.w3.org/2001/XMLSchema-instance" - -# The "class of instrument" element a must carry (FundsXML models -# this as a choice). We store its name in position.kind and, for the kinds -# that require a quantity child, the numeric quantity in position.kind_qty. -# WHY: without this we could not regenerate a schema-valid Position (an empty -# is invalid — it needs ). Kinds not in QTY_ELEM -# (FXForward/Swap/Repo/RealEstate/CallMoney/...) are valid as an empty element. -POSITION_KINDS = {"Equity", "Bond", "ShareClass", "Warrant", "Certificate", - "Option", "Future", "FXForward", "Swap", "Repo", - "RealEstate", "CallMoney", "Account", "Generic"} -QTY_ELEM = {"Equity": "Units", "Warrant": "Units", "Certificate": "Units", - "Bond": "Nominal", "ShareClass": "Shares", - "Option": "Contracts", "Future": "Contracts"} - - -# --------------------------------------------------------------------------- -# small helpers -# --------------------------------------------------------------------------- -def _text(node, path): - """First matching element's text, or None. Bare names (no namespace).""" - r = node.xpath(path) - return r[0].text if r and r[0].text is not None else None - - -def _num(v): - return float(v) if v not in (None, "") else None - - -def _el(parent, tag, text=None, **attrs): - """Append a child element; the one XML-building primitive used throughout - the exporter so the emitted structure stays consistent and reviewable.""" - e = etree.SubElement(parent, tag) - for k, v in attrs.items(): - e.set(k, str(v)) - if text is not None: - e.text = str(text) - return e - - -def connect(db): - con = sqlite3.connect(db) - con.execute("PRAGMA foreign_keys = ON") # enforce the FK graph - con.row_factory = sqlite3.Row - return con - - -# --------------------------------------------------------------------------- -# init : create the schema -# --------------------------------------------------------------------------- -def init(db): - con = connect(db) - con.executescript(DDL.read_text()) - con.commit() - con.close() - - -# --------------------------------------------------------------------------- -# import : FundsXML -> relational rows -# -# We walk the document in its natural order and assign 1-based ordinals -# (fund_seq / portfolio_seq / position_seq). Those ordinals are what make the -# export deterministic and therefore comparable to the original file. -# --------------------------------------------------------------------------- -def do_import(db, xml_path): - parser = etree.XMLParser(resolve_entities=False, no_network=True, - load_dtd=False, huge_tree=False) - doc = etree.parse(xml_path, parser) - cd = doc.xpath("/FundsXML4/ControlData")[0] - document_id = _text(cd, "UniqueDocumentID") - - con = connect(db) - - # ControlData -> one `document` row (so it round-trips exactly instead of - # being hard-coded by the exporter). - con.execute( - "INSERT INTO document VALUES (?,?,?,?,?,?,?,?,?)", - (document_id, _text(cd, "DocumentGenerated"), _text(cd, "Version"), - _text(cd, "ContentDate"), _text(cd, "DataOperation"), - _text(cd, "DataSupplier/SystemCountry"), - _text(cd, "DataSupplier/Short"), - _text(cd, "DataSupplier/Name"), - _text(cd, "DataSupplier/Type"))) - - # Every (not just the first). - for fund_seq, fund in enumerate(doc.xpath("/FundsXML4/Funds/Fund"), start=1): - ccy = _text(fund, "Currency") - tav = fund.xpath("FundDynamicData/TotalAssetValues/TotalAssetValue")[0] - con.execute( - "INSERT INTO fund VALUES (?,?,?,?,?,?,?,?)", - (document_id, fund_seq, _text(fund, "Identifiers/LEI"), - _text(fund, "Names/OfficialName"), ccy, - _text(fund, "SingleFundFlag"), _text(tav, "NavDate"), - float(tav.xpath( - f"TotalNetAssetValue/Amount[@ccy='{ccy}']")[0].text))) - - # Share classes of THIS fund. - for sc in fund.xpath("SingleFund/ShareClasses/ShareClass"): - navf = sc.xpath("TotalAssetValues/TotalAssetValue/" - f"TotalNetAssetValue/Amount[@ccy='{ccy}']") - con.execute( - "INSERT INTO share_class VALUES (?,?,?,?,?,?,?,?)", - (document_id, fund_seq, _text(sc, "Identifiers/ISIN"), - _text(sc, "Names/OfficialName"), _text(sc, "Currency"), - _num(_text(sc, "Prices/Price/NavPrice")), - _num(navf[0].text) if navf else None, - _num(_text(sc, "TotalAssetValues/TotalAssetValue/" - "SharesOutstanding")))) - - # Every of this fund, and every in it. - for p_seq, port in enumerate( - fund.xpath("FundDynamicData/Portfolios/Portfolio"), start=1): - con.execute("INSERT INTO portfolio VALUES (?,?,?,?)", - (document_id, fund_seq, p_seq, _text(port, "NavDate"))) - for pos_seq, pos in enumerate( - port.xpath("Positions/Position"), start=1): - # The Position's instrument-class child (Equity/Bond/...). - kinds = [c.tag for c in pos if c.tag in POSITION_KINDS] - kind = kinds[0] if kinds else None - qty = (_num(_text(pos, f"{kind}/{QTY_ELEM[kind]}")) - if kind in QTY_ELEM else None) - con.execute( - "INSERT INTO position VALUES (?,?,?,?,?,?,?,?,?,?,?)", - (document_id, fund_seq, p_seq, pos_seq, - _text(pos, "UniqueID"), _text(pos, "Identifiers/ISIN"), - _text(pos, "Currency"), - float(pos.xpath( - f"TotalValue/Amount[@ccy='{ccy}']")[0].text), - float(_text(pos, "TotalPercentage")), kind, qty)) - - # AssetMasterData is shared by all funds (document-scoped). - for a in doc.xpath("/FundsXML4/AssetMasterData/Asset"): - con.execute( - "INSERT INTO asset VALUES (?,?,?,?,?,?,?)", - (document_id, _text(a, "UniqueID"), _text(a, "Identifiers/ISIN"), - _text(a, "Name"), _text(a, "AssetType"), - _text(a, "Currency"), _text(a, "Country"))) - - con.commit() - con.close() - return document_id - - -# --------------------------------------------------------------------------- -# export : relational rows -> FundsXML -# -# Emits a fixed, canonical structure in schema order. Constants that the model -# does not store (TotalAssetNature=OFFICIAL, Price ActionCode=C, -# PriceNature=OFFICIAL) are reproduced verbatim; the round-trip fixture uses -# the same constants, so xml_equiv.py sees the files as equal. -# --------------------------------------------------------------------------- -def export(db, document_id, out_path): - con = connect(db) - d = con.execute("SELECT * FROM document WHERE document_id=?", - (document_id,)).fetchone() - if d is None: - raise SystemExit(f"no document {document_id!r} in {db}") - - root = etree.Element("FundsXML4", nsmap={"xsi": XSI}) - root.set(f"{{{XSI}}}noNamespaceSchemaLocation", SCHEMA_URL) - - cd = _el(root, "ControlData") - _el(cd, "UniqueDocumentID", d["document_id"]) - # Regenerate the timestamp; xml_equiv.py ignores its value by design. - _el(cd, "DocumentGenerated", - d["generated"] or datetime.now(timezone.utc) - .strftime("%Y-%m-%dT%H:%M:%S")) - if d["version"]: # absent for FundsXML 4.0.0 - _el(cd, "Version", d["version"]) - _el(cd, "ContentDate", d["content_date"]) - ds = _el(cd, "DataSupplier") - _el(ds, "SystemCountry", d["supplier_country"]) - _el(ds, "Short", d["supplier_short"]) - _el(ds, "Name", d["supplier_name"]) - _el(ds, "Type", d["supplier_type"]) - _el(cd, "DataOperation", d["data_operation"]) - - funds = con.execute( - "SELECT * FROM fund WHERE document_id=? ORDER BY fund_seq", - (document_id,)).fetchall() - funds_el = _el(root, "Funds") - for f in funds: - ccy = f["currency"] - fund = _el(funds_el, "Fund") - if f["lei"]: - _el(_el(fund, "Identifiers"), "LEI", f["lei"]) - _el(_el(fund, "Names"), "OfficialName", f["official_name"]) - _el(fund, "Currency", ccy) - if f["single_fund_flag"]: - _el(fund, "SingleFundFlag", f["single_fund_flag"]) - - fdd = _el(fund, "FundDynamicData") - tav = _el(_el(_el(fdd, "TotalAssetValues"), "TotalAssetValue"), - "NavDate", f["nav_date"]).getparent() - _el(tav, "TotalAssetNature", "OFFICIAL") - _el(_el(tav, "TotalNetAssetValue"), "Amount", - f'{f["total_nav"]:.2f}', ccy=ccy) - - ports = _el(fdd, "Portfolios") - for pf in con.execute( - "SELECT * FROM portfolio WHERE document_id=? AND fund_seq=? " - "ORDER BY portfolio_seq", (document_id, f["fund_seq"])): - pe = _el(ports, "Portfolio") - _el(pe, "NavDate", pf["nav_date"]) - poss = _el(pe, "Positions") - for p in con.execute( - "SELECT * FROM position WHERE document_id=? AND fund_seq=? " - "AND portfolio_seq=? ORDER BY position_seq", - (document_id, f["fund_seq"], pf["portfolio_seq"])): - pos = _el(poss, "Position") - _el(pos, "UniqueID", p["unique_id"]) - if p["isin"]: - _el(_el(pos, "Identifiers"), "ISIN", p["isin"]) - if p["currency"]: - _el(pos, "Currency", p["currency"]) - _el(_el(pos, "TotalValue"), "Amount", - f'{p["value_fund_ccy"]:.2f}', ccy=ccy) - _el(pos, "TotalPercentage", f'{p["percentage"]:.2f}') - kind = p["kind"] if p["kind"] in POSITION_KINDS else "Generic" - ke = _el(pos, kind) - if kind in QTY_ELEM and p["kind_qty"] is not None: - _el(ke, QTY_ELEM[kind], f'{p["kind_qty"]:.2f}') - - # Share classes (authored in ISIN order in the fixture, so ORDER BY - # isin makes export child-order match the original). - scs = con.execute( - "SELECT * FROM share_class WHERE document_id=? AND fund_seq=? " - "ORDER BY isin", (document_id, f["fund_seq"])).fetchall() - if scs: - sce = _el(_el(fund, "SingleFund"), "ShareClasses") - for sc in scs: - x = _el(sce, "ShareClass") - _el(_el(x, "Identifiers"), "ISIN", sc["isin"]) - if sc["official_name"]: - _el(_el(x, "Names"), "OfficialName", sc["official_name"]) - _el(x, "Currency", sc["currency"]) - if sc["nav_price"] is not None: - pr = _el(_el(x, "Prices"), "Price") - _el(pr, "ActionCode", "C") - _el(pr, "NavDate", f["nav_date"]) - _el(pr, "PriceCurrency", sc["currency"]) - _el(pr, "PriceNature", "OFFICIAL") - _el(pr, "NavPrice", f'{sc["nav_price"]:.2f}') - if sc["nav_fund_ccy"] is not None: - t = _el(_el(x, "TotalAssetValues"), "TotalAssetValue") - _el(t, "NavDate", f["nav_date"]) - _el(t, "TotalAssetNature", "OFFICIAL") - _el(_el(t, "TotalNetAssetValue"), "Amount", - f'{sc["nav_fund_ccy"]:.2f}', ccy=ccy) - if sc["shares_outstanding"] is not None: - _el(t, "SharesOutstanding", - f'{sc["shares_outstanding"]:.0f}') - - assets = con.execute( - "SELECT * FROM asset WHERE document_id=? ORDER BY unique_id", - (document_id,)).fetchall() - if assets: - amd = _el(root, "AssetMasterData") - for a in assets: - ae = _el(amd, "Asset") - _el(ae, "UniqueID", a["unique_id"]) - if a["isin"]: - _el(_el(ae, "Identifiers"), "ISIN", a["isin"]) - _el(ae, "Currency", a["currency"]) - if a["country"]: - _el(ae, "Country", a["country"]) - _el(ae, "Name", a["name"]) - _el(ae, "AssetType", a["asset_type"]) - con.close() - - etree.ElementTree(root).write(out_path, xml_declaration=True, - encoding="UTF-8", pretty_print=True) - - -# --------------------------------------------------------------------------- -# cli -# --------------------------------------------------------------------------- -def main() -> int: - a = sys.argv[1:] - if not a: - print(__doc__, file=sys.stderr) - return 2 - cmd = a[0] - if cmd == "init": - init(a[1]) - elif cmd == "import": - print("imported document_id:", do_import(a[1], a[2])) - elif cmd == "export": - export(a[1], a[2], a[3]) - print("wrote", a[3]) - elif cmd == "roundtrip": - # import then export THROUGH the DB (the user's required test): the - # data exported is exactly the data the import wrote. - with tempfile.TemporaryDirectory() as t: - db = str(Path(t) / "rt.db") - init(db) - doc_id = do_import(db, a[1]) - export(db, doc_id, a[2]) - print(f"round-trip ok: {a[1]} -> DB -> {a[2]} (doc {doc_id})") - else: - print(f"unknown command {cmd!r}", file=sys.stderr) - return 2 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/Database_Integration/python/import_fundsxml.py b/Database_Integration/python/import_fundsxml.py new file mode 100644 index 0000000..d2e0857 --- /dev/null +++ b/Database_Integration/python/import_fundsxml.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# ============================================================================= +# IMPORT — FundsXML file -> relational database (Python 3, SQLite). +# +# Standalone, copy-me example of ONE direction only: reading a FundsXML file +# into the relational schema. The reverse (DB -> FundsXML) is a separate +# program, export_fundsxml.py. Over-commented so it doubles as documentation. +# +# DB SCHEMA ../ddl/schema.sql (document -> fund -> portfolio -> position; +# share_class per fund; asset document-scoped). This program CREATES the +# schema in a fresh SQLite DB and then loads the file. +# +# RUN +# python3 import_fundsxml.py fx.db \ +# ../../FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml +# # then, separately: python3 export_fundsxml.py fx.db out.xml +# +# DEPENDENCIES Python stdlib `sqlite3` + `lxml`. +# +# FUNDSXML ASSUMPTIONS (non-obvious) +# * FundsXML 4.x has NO XML namespace -> XPath uses bare element names. +# * A document may hold MANY , each MANY , each MANY +# . We iterate all of them and record 1-based *_seq ordinals so +# a later export reproduces the document order exactly. +# * Positions link to AssetMasterData by a shared -> `asset` is +# document-scoped, not per-fund. +# +# SECURITY the XML parser disables network access and entity resolution +# (XXE / billion laughs). FundsXML needs no DTD/external entities. +# ============================================================================= +import sqlite3 +import sys +from pathlib import Path + +from lxml import etree + +DDL = Path(__file__).resolve().parents[1] / "ddl" / "schema.sql" + +# The Position "class of instrument" element (FundsXML models this as a +# choice). We store its name and, for the kinds that require a quantity child, +# the numeric quantity, so the export can rebuild a schema-valid Position. +POSITION_KINDS = {"Equity", "Bond", "ShareClass", "Warrant", "Certificate", + "Option", "Future", "FXForward", "Swap", "Repo", + "RealEstate", "CallMoney", "Account", "Generic"} +QTY_ELEM = {"Equity": "Units", "Warrant": "Units", "Certificate": "Units", + "Bond": "Nominal", "ShareClass": "Shares", + "Option": "Contracts", "Future": "Contracts"} + + +def _text(node, path): + """First matching element's text, or None (bare names; no namespace).""" + r = node.xpath(path) + return r[0].text if r and r[0].text is not None else None + + +def _num(v): + return float(v) if v not in (None, "") else None + + +def main() -> int: + if len(sys.argv) != 3: + print("usage: import_fundsxml.py ", + file=sys.stderr) + return 2 + db, xml_path = sys.argv[1], sys.argv[2] + + con = sqlite3.connect(db) + con.execute("PRAGMA foreign_keys = ON") + # Create the relational schema (this example targets a fresh DB). + con.executescript(DDL.read_text()) + + # Parse hardened: no network, no entity resolution, no huge-tree blowups. + parser = etree.XMLParser(resolve_entities=False, no_network=True, + load_dtd=False, huge_tree=False) + doc = etree.parse(xml_path, parser) + + cd = doc.xpath("/FundsXML4/ControlData")[0] + document_id = _text(cd, "UniqueDocumentID") + + # ControlData -> one `document` row, so it round-trips exactly instead of + # being hard-coded by the exporter. + con.execute( + "INSERT INTO document VALUES (?,?,?,?,?,?,?,?,?)", + (document_id, _text(cd, "DocumentGenerated"), _text(cd, "Version"), + _text(cd, "ContentDate"), _text(cd, "DataOperation"), + _text(cd, "DataSupplier/SystemCountry"), + _text(cd, "DataSupplier/Short"), + _text(cd, "DataSupplier/Name"), + _text(cd, "DataSupplier/Type"))) + + # Every (not just the first) gets a 1-based fund_seq. + for fund_seq, fund in enumerate(doc.xpath("/FundsXML4/Funds/Fund"), + start=1): + ccy = _text(fund, "Currency") + tav = fund.xpath("FundDynamicData/TotalAssetValues/TotalAssetValue")[0] + con.execute( + "INSERT INTO fund VALUES (?,?,?,?,?,?,?,?)", + (document_id, fund_seq, _text(fund, "Identifiers/LEI"), + _text(fund, "Names/OfficialName"), ccy, + _text(fund, "SingleFundFlag"), _text(tav, "NavDate"), + float(tav.xpath( + f"TotalNetAssetValue/Amount[@ccy='{ccy}']")[0].text))) + + for sc in fund.xpath("SingleFund/ShareClasses/ShareClass"): + navf = sc.xpath("TotalAssetValues/TotalAssetValue/" + f"TotalNetAssetValue/Amount[@ccy='{ccy}']") + con.execute( + "INSERT INTO share_class VALUES (?,?,?,?,?,?,?,?)", + (document_id, fund_seq, _text(sc, "Identifiers/ISIN"), + _text(sc, "Names/OfficialName"), _text(sc, "Currency"), + _num(_text(sc, "Prices/Price/NavPrice")), + _num(navf[0].text) if navf else None, + _num(_text(sc, "TotalAssetValues/TotalAssetValue/" + "SharesOutstanding")))) + + for p_seq, port in enumerate( + fund.xpath("FundDynamicData/Portfolios/Portfolio"), start=1): + con.execute("INSERT INTO portfolio VALUES (?,?,?,?)", + (document_id, fund_seq, p_seq, + _text(port, "NavDate"))) + for pos_seq, pos in enumerate( + port.xpath("Positions/Position"), start=1): + kinds = [c.tag for c in pos if c.tag in POSITION_KINDS] + kind = kinds[0] if kinds else None + qty = (_num(_text(pos, f"{kind}/{QTY_ELEM[kind]}")) + if kind in QTY_ELEM else None) + con.execute( + "INSERT INTO position VALUES (?,?,?,?,?,?,?,?,?,?,?)", + (document_id, fund_seq, p_seq, pos_seq, + _text(pos, "UniqueID"), _text(pos, "Identifiers/ISIN"), + _text(pos, "Currency"), + float(pos.xpath( + f"TotalValue/Amount[@ccy='{ccy}']")[0].text), + float(_text(pos, "TotalPercentage")), kind, qty)) + + # AssetMasterData is shared by all funds (document-scoped). + for a in doc.xpath("/FundsXML4/AssetMasterData/Asset"): + con.execute( + "INSERT INTO asset VALUES (?,?,?,?,?,?,?)", + (document_id, _text(a, "UniqueID"), _text(a, "Identifiers/ISIN"), + _text(a, "Name"), _text(a, "AssetType"), + _text(a, "Currency"), _text(a, "Country"))) + + con.commit() + con.close() + print("imported document_id:", document_id) + return 0 + + +if __name__ == "__main__": + sys.exit(main())