diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c31b5c..4895c91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,32 @@ jobs: fi echo "xsd-invalid correctly rejected" + # --------------------------------------------------------------------- + # Python examples — one idiomatic, OS-agnostic install (venv + editable + # pyproject) replaces the old ad-hoc `pip install lxml`. This also + # exposes the `fundsxml_schema` resolver module to every Python script. + # --------------------------------------------------------------------- + - name: Python - venv & install (pyproject) + run: | + python -m venv .venv + .venv/bin/pip install --quiet --upgrade pip + .venv/bin/pip install --quiet -e . + .venv/bin/python -c "import lxml, saxonche, fundsxml_schema; print('python deps OK')" + + - name: Python - XSD validation (in-language schema resolve) + run: | + set -e + V=".venv/bin/python XSD_Validation/python/validate.py" + # Cache is warm from the xmllint step: exercises the resolver's + # cache-hit path and the $FUNDSXML_SCHEMA_DIR env override. + $V 4.2.9 FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml + FUNDSXML_SCHEMA_DIR="$PWD/.schema-cache/4.1.0" \ + $V 4.1.0 FundsXML_Files/4.1.0/positions/Equity-Fund_Positions.xml + if $V 4.2.9 tests/fixtures/invalid/xsd-invalid_Positions.xml; then + echo "::error::xsd-invalid unexpectedly validated (Python)"; exit 1 + fi + echo "Python validate.py: positive ok, negative correctly rejected" + # --------------------------------------------------------------------- # Java examples — built & run via the committed Maven Wrapper. The first # ./mvnw bootstraps Maven itself, then resolves all deps from Central. @@ -155,11 +181,11 @@ jobs: - name: DB integration - multi-fund import+export in 4 languages run: | set -e - python3 -m pip install --quiet lxml + PY=.venv/bin/python FX=FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml MX=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml DOC=FUNDSXML_MULTI_1 - EQ="python3 Database_Integration/tools/xml_equiv.py" + EQ="$PY Database_Integration/tools/xml_equiv.py" XSD=".schema-cache/4.2.9/FundsXML.xsd" # The multi-fund fixture must itself be schema-valid. @@ -169,8 +195,8 @@ jobs: # exported file == input file (and is schema-valid). # Python - python3 Database_Integration/python/import_fundsxml.py py.db "$FX" - python3 Database_Integration/python/export_fundsxml.py py.db "$DOC" py.xml + $PY Database_Integration/python/import_fundsxml.py py.db "$FX" + $PY Database_Integration/python/export_fundsxml.py py.db "$DOC" py.xml xmllint --noout --nonet --schema "$XSD" py.xml $EQ "$FX" py.xml @@ -198,19 +224,19 @@ jobs: $EQ py.xml java.xml && $EQ py.xml js.xml && $EQ py.xml cs.xml # Single-fund sample still imports+exports (lossy -> XSD-valid only). - python3 Database_Integration/python/import_fundsxml.py mx.db "$MX" - python3 Database_Integration/python/export_fundsxml.py mx.db FUNDSXML_FILE_1 mx.xml + $PY Database_Integration/python/import_fundsxml.py mx.db "$MX" + $PY Database_Integration/python/export_fundsxml.py mx.db FUNDSXML_FILE_1 mx.xml xmllint --noout --nonet --schema "$XSD" mx.xml echo "multi-fund import+export equivalent in python/java/javascript/csharp" - name: Large-file - streaming aggregate / split / delta (constant memory) run: | set -e - python3 -m pip install --quiet lxml + PY=.venv/bin/python P=Large_File_Processing/python - python3 $P/make_large_sample.py big.xml 30000 + $PY $P/make_large_sample.py big.xml 30000 xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd big.xml - python3 $P/stream_aggregate.py big.xml | tee agg.txt + $PY $P/stream_aggregate.py big.xml | tee agg.txt grep -q '^positions : 30000$' agg.txt grep -q '^sum value (EUR): 30000000.00$' agg.txt # Java StreamAggregate (pure JDK) under a small heap proves the @@ -218,24 +244,24 @@ jobs: MAVEN_OPTS=-Xmx64m ./mvnw -q -B -pl Large_File_Processing/java exec:java \ -Dexec.args="big.xml" | tee aggj.txt grep -q '^positions : 30000$' aggj.txt - python3 $P/split.py big.xml chunks/ 10000 + $PY $P/split.py big.xml chunks/ 10000 test "$(ls chunks/chunk-*.xml | wc -l)" = "3" xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd chunks/chunk-0001.xml - python3 $P/delta_diff.py big.xml big.xml # identical -> exit 0 + $PY $P/delta_diff.py big.xml big.xml # identical -> exit 0 - name: Data binding / JSON - round-trip + native Java binding run: | set -e - python3 -m pip install --quiet lxml + PY=.venv/bin/python SRC=FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml - python3 Data_Binding_JSON/python/fundsxml_json.py roundtrip "$SRC" rj.xml + $PY Data_Binding_JSON/python/fundsxml_json.py roundtrip "$SRC" rj.xml xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd rj.xml # Multi-fund JSON round-trip is lossless -> must be xml_equiv-equal. MF=FundsXML_Files/4.2.9/positions/Multi-Fund_Positions.xml - python3 Data_Binding_JSON/python/fundsxml_json.py roundtrip "$MF" rjm.xml + $PY Data_Binding_JSON/python/fundsxml_json.py roundtrip "$MF" rjm.xml xmllint --noout --nonet --schema .schema-cache/4.2.9/FundsXML.xsd rjm.xml - python3 Database_Integration/tools/xml_equiv.py "$MF" rjm.xml - python3 - "$SRC" rj.xml <<'PY' + $PY Database_Integration/tools/xml_equiv.py "$MF" rjm.xml + $PY - "$SRC" rj.xml <<'PY' import re, sys o, r = (open(p).read() for p in sys.argv[1:3]) nav = lambda x: re.search(r'\s*([0-9.]+)', x).group(1) diff --git a/.gitignore b/.gitignore index b2a01d4..ebb9aef 100644 --- a/.gitignore +++ b/.gitignore @@ -41,9 +41,12 @@ fund.json *.db *.svrl -# Python bytecode cache +# Python virtualenv (created by `python -m venv .venv`) + bytecode cache + +# editable-install metadata. Deps come from pyproject.toml via pip. +.venv/ __pycache__/ *.pyc +*.egg-info/ # Node example deps (Database_Integration/javascript) — restored via npm install node_modules/ @@ -52,3 +55,6 @@ package-lock.json # .NET build output (Database_Integration/csharp) — restored via dotnet build bin/ obj/ + +# Claude Code local session state +.claude/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4c8dbe4..675308d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,15 +55,18 @@ Please read this before opening a pull request. Run what you changed and confirm it actually works — no "should pass" claims. -The Java examples build standalone via the committed Maven Wrapper (`./mvnw`, -or `mvnw.cmd` on Windows) — dependencies come from Maven Central, there is no -`fetch-tools.sh` and no `.lib/`. +Java examples build standalone via the committed Maven Wrapper (`./mvnw`, or +`mvnw.cmd` on Windows). Python examples install once into a venv from +`pyproject.toml` and resolve the XSD themselves. Neither needs `fetch-tools.sh` +(gone) and Python no longer needs `fetch-schema.sh`. ```bash -tools/fetch-schema.sh 4.2.9 # XSD cache for the xmllint/Python steps - # (the Java examples also resolve the - # XSD themselves; see XsdValidate) +# Python stack (cross-platform; Windows: .venv\Scripts\activate) +python -m venv .venv && . .venv/bin/activate && pip install -e . +python XSD_Validation/python/validate.py 4.2.9 .xml # self-resolves the XSD +# xmllint still uses the cached schema (CLI stack, removed in a later phase) +tools/fetch-schema.sh 4.2.9 xmllint --noout --schema .schema-cache/4.2.9/FundsXML.xsd .xml # Schematron via the Maven Wrapper (positive sample -> exit 0) @@ -71,10 +74,10 @@ xmllint --noout --schema .schema-cache/4.2.9/FundsXML.xsd .xml compile exec:java \ -Dexec.args="Schematron_DataQuality_Checks/Basic_Checks/basic_checks.sch .xml" -# DB round-trip example: -python3 Database_Integration/python/import_fundsxml.py fx.db .xml -python3 Database_Integration/python/export_fundsxml.py fx.db out.xml -python3 Database_Integration/tools/xml_equiv.py .xml out.xml +# DB round-trip example (venv python): +python Database_Integration/python/import_fundsxml.py fx.db .xml +python Database_Integration/python/export_fundsxml.py fx.db out.xml +python Database_Integration/tools/xml_equiv.py .xml out.xml ``` Toolchain notes: the SchXslt CLI jar bundles its own Saxon — do **not** add the diff --git a/README.md b/README.md index cec5324..d1340db 100644 --- a/README.md +++ b/README.md @@ -133,9 +133,10 @@ xsltproc XSLT_DataQuality_Checks/Enhanced_Check/FundsXML_CompleteDQReport_HTML.x ### Option 3: Python ```bash -# Install dependencies -pip install lxml # XSLT 1.0 -pip install saxonche # XSLT 2.0/3.0 +# Install all Python deps once (lxml + saxonche), cross-platform: +python -m venv .venv +. .venv/bin/activate # Windows: .venv\Scripts\activate +pip install -e . # see pyproject.toml # Run transformation python -c " diff --git a/Schematron_DataQuality_Checks/Basic_Checks/invocation/README.md b/Schematron_DataQuality_Checks/Basic_Checks/invocation/README.md index 91650f4..0c7e92c 100644 --- a/Schematron_DataQuality_Checks/Basic_Checks/invocation/README.md +++ b/Schematron_DataQuality_Checks/Basic_Checks/invocation/README.md @@ -38,7 +38,7 @@ exits 1 on any failed-assert (including warnings). | Stack | File | Runnable on this box | |-------|------|----------------------| | Java (native) | [`SchematronValidate.java`](SchematronValidate.java) | ✅ verified (SchXslt Java API, via Maven Wrapper) | -| Python | [`validate_schematron.py`](validate_schematron.py) | needs `pip install saxonche` | +| Python | [`validate_schematron.py`](validate_schematron.py) | saxonche via repo venv (`pip install -e .`); SchXslt jar via `$FUNDSXML_SCHXSLT_JAR` or Maven local repo — reference variant | | .NET/C# | [`SchematronValidate.cs`](SchematronValidate.cs) | needs .NET SDK + `SaxonHE` package | | shared | [`svrl-summary.py`](svrl-summary.py) | ✅ classifier used by all + CI | diff --git a/Schematron_DataQuality_Checks/Basic_Checks/invocation/validate_schematron.py b/Schematron_DataQuality_Checks/Basic_Checks/invocation/validate_schematron.py index 18e0488..49a3a2c 100644 --- a/Schematron_DataQuality_Checks/Basic_Checks/invocation/validate_schematron.py +++ b/Schematron_DataQuality_Checks/Basic_Checks/invocation/validate_schematron.py @@ -6,17 +6,20 @@ basic_checks.sch uses queryBinding="xslt2", so an XSLT 2.0 processor is required. `lxml` only does XSLT 1.0 and CANNOT run this ruleset — saxonche -(`pip install saxonche`) embeds Saxon's XSLT 3.0 engine. +(installed via the repo's pyproject.toml: `pip install -e .`) embeds Saxon's +XSLT 3.0 engine. -The SchXslt pipeline stylesheets are reused straight out of the SchXslt CLI jar -(this Python stack will resolve it via pyproject.toml once migrated; the Java -Schematron example already runs standalone via the Maven Wrapper). The whole -`xslt/` tree is extracted -to a temp dir so the pipeline's relative xsl:import/include resolve, then: +Reference variant: SchXslt has no PyPI package, so the SchXslt pipeline +stylesheets are reused straight out of the SchXslt CLI jar, located via +$FUNDSXML_SCHXSLT_JAR or the Maven local repo (see _find_schxslt_jar). The +Java Schematron example is the verified, fully standalone path (Maven +Wrapper). The whole `xslt/` tree is extracted to a temp dir so the +pipeline's relative xsl:import/include resolve, then: 1) compile the .sch into an SVRL stylesheet, 2) apply it to the instance to get SVRL, 3) classify with the shared svrl-summary.py. """ +import os import subprocess import sys import tempfile @@ -25,7 +28,28 @@ HERE = Path(__file__).resolve().parent REPO_ROOT = HERE.parents[2] -CLI_JAR = REPO_ROOT / ".lib" / "schxslt-cli-1.10.1.jar" + +# The SchXslt pipeline stylesheets are reused straight out of the SchXslt CLI +# jar. There is no PyPI distribution of SchXslt, so this (reference) Python +# stack locates the jar standalone, in order: +# 1. $FUNDSXML_SCHXSLT_JAR (explicit path) +# 2. the Maven local repo — the Java Schematron module already declares +# name.dmaus.schxslt:cli:1.10.1, so `./mvnw -pl Schematron_DataQuality_ +# Checks/Basic_Checks/invocation compile` populates ~/.m2 with it. +_SCHXSLT_VERSION = "1.10.1" +_M2 = Path(os.environ.get("MAVEN_REPO_LOCAL", + Path.home() / ".m2" / "repository")) + + +def _find_schxslt_jar() -> Path: + env = os.environ.get("FUNDSXML_SCHXSLT_JAR") + if env: + return Path(env) + return (_M2 / "name" / "dmaus" / "schxslt" / "cli" / _SCHXSLT_VERSION + / f"cli-{_SCHXSLT_VERSION}.jar") + + +CLI_JAR = _find_schxslt_jar() def main() -> int: @@ -44,10 +68,13 @@ def main() -> int: return 2 if not CLI_JAR.is_file(): - print("SchXslt jar missing (this Python stack is migrated to a build " - "system in a later phase; the Java example already runs " - "standalone: ./mvnw -pl Schematron_DataQuality_Checks/" - "Basic_Checks/invocation compile exec:java)", file=sys.stderr) + print(f"SchXslt jar not found at {CLI_JAR}.\n" + "Set $FUNDSXML_SCHXSLT_JAR, or populate the Maven local repo " + "once with:\n" + " ./mvnw -q -pl Schematron_DataQuality_Checks/Basic_Checks/" + "invocation compile\n" + "(the Java Schematron example runs fully standalone via the " + "Maven Wrapper and is the verified path.)", file=sys.stderr) return 2 with tempfile.TemporaryDirectory() as tmp: diff --git a/XQuery_Examples/README.md b/XQuery_Examples/README.md index 0986b33..4f10743 100644 --- a/XQuery_Examples/README.md +++ b/XQuery_Examples/README.md @@ -19,7 +19,7 @@ no XML namespace — queries use bare element names, so they work unchanged on t | Stack | Entry point | Status | |-------|-------------|--------| | Java (s9api, no JAXB) | [`invocation/RunXQuery.java`](invocation/RunXQuery.java) | ✅ verified (via Maven Wrapper) | -| Python | [`invocation/run_xquery.py`](invocation/run_xquery.py) | needs `pip install saxonche` | +| Python | [`invocation/run_xquery.py`](invocation/run_xquery.py) | standalone via repo venv (`pip install -e .`) | | BaseX | see below | needs BaseX install | The Java runner is standalone & cross-platform via the committed Maven Wrapper diff --git a/XSD_Validation/README.md b/XSD_Validation/README.md index 1e43945..66d49a2 100644 --- a/XSD_Validation/README.md +++ b/XSD_Validation/README.md @@ -21,12 +21,20 @@ Two realities every example must deal with: 2. **Relative import.** From release 4.2.9 on, `FundsXML.xsd` imports `xmldsig-core-schema.xsd` via a *relative* path; both files must sit together. -`tools/fetch-schema.sh ` resolves both (proxy-aware via curl) and -materializes the released schema into `.schema-cache//`. The examples -validate against that materialized release. Run it once up front: +**Schema resolution (same convention in every stack):** +`$FUNDSXML_SCHEMA_DIR` (a hand-placed copy — offline / corporate-network +escape hatch) → `.schema-cache//` → download from the official +GitHub release (302-aware; also pulls the imported `xmldsig-core-schema.xsd`), +caching into `.schema-cache/`. The official release stays the source of truth +— no committed catalog. + +The **Java** (`XsdValidate`) and **Python** (`validate.py`) examples do this +themselves — standalone, cross-platform, no prior step. `tools/fetch-schema.sh` +still seeds the cache for the CLI/xmllint and (until their phases land) +.NET/PowerShell stacks: ```bash -tools/fetch-schema.sh 4.2.9 +tools/fetch-schema.sh 4.2.9 # only needed for the CLI/.NET/PS stacks ``` ## Security @@ -40,8 +48,8 @@ Every example disables external entity resolution / DTD loading | Stack | Script | API | Runnable on this box | |-------|--------|-----|----------------------| | CLI | [`cli/validate.sh`](cli/validate.sh) | `xmllint` (+ Saxon note) | ✅ | -| Python | [`python/validate.py`](python/validate.py) | `lxml.etree.XMLSchema` | ✅ | -| Java | [`java/XsdValidate.java`](java/XsdValidate.java) | `javax.xml.validation` | ✅ (single-file, JDK 11+) | +| Python | [`python/validate.py`](python/validate.py) | `lxml.etree.XMLSchema` | ✅ standalone (`pip install -e .`) | +| Java | [`java/XsdValidate.java`](java/XsdValidate.java) | `javax.xml.validation` | ✅ standalone (`./mvnw`) | | .NET/C# | [`dotnet/XsdValidate.cs`](dotnet/XsdValidate.cs) | `XmlSchemaSet` | needs .NET SDK | | PowerShell | [`powershell/Validate-FundsXml.ps1`](powershell/Validate-FundsXml.ps1) | `System.Xml.Schema` | needs PowerShell | @@ -50,8 +58,15 @@ invalid, prints errors to stderr. ## Quick check (positive + negative) +Python (standalone — resolves the schema itself; `pip install -e .` once, see +the repo `pyproject.toml`): + ```bash -tools/fetch-schema.sh 4.2.9 -XSD_Validation/cli/validate.sh 4.2.9 FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml # exit 0 -XSD_Validation/cli/validate.sh 4.2.9 tests/fixtures/invalid/xsd-invalid_Positions.xml # exit 1 +python -m venv .venv && . .venv/bin/activate && pip install -e . # Windows: .venv\Scripts\activate +python XSD_Validation/python/validate.py 4.2.9 FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml # exit 0 +python XSD_Validation/python/validate.py 4.2.9 tests/fixtures/invalid/xsd-invalid_Positions.xml # exit 1 ``` + +Java (standalone): `./mvnw -q -pl XSD_Validation/java compile exec:java -Dexec.args="4.2.9 "` +(`mvnw.cmd` on Windows). The CLI/`xmllint` stack still uses +`tools/fetch-schema.sh 4.2.9` until its phase lands. diff --git a/XSD_Validation/python/validate.py b/XSD_Validation/python/validate.py index 3f18b0a..a7abdd5 100644 --- a/XSD_Validation/python/validate.py +++ b/XSD_Validation/python/validate.py @@ -1,34 +1,27 @@ #!/usr/bin/env python3 """XSD validation in Python via lxml. -Usage: python XSD_Validation/python/validate.py +Standalone & cross-platform — no bash, no prior tool step (works on Windows). +After `pip install -e .` (see pyproject.toml): + + python XSD_Validation/python/validate.py Exit: 0 = valid, 1 = invalid, 2 = usage/setup error -Validates against the official released schema, materialized locally by -tools/fetch-schema.sh (handles the GitHub 302 redirect and the relative -xmldsig-core-schema.xsd import that FundsXML 4.2.9+ requires). +The official released schema is obtained by this program itself via the shared +`fundsxml_schema` resolver: $FUNDSXML_SCHEMA_DIR (offline/corporate escape +hatch) -> .schema-cache/ -> download from the official GitHub release +(following the 302; fetching the relative xmldsig-core-schema.xsd sibling that +FundsXML 4.2.9+ imports). The official release stays the source of truth. Security: the XML parser is hardened against XXE / entity-expansion (no_network=True, resolve_entities=False, no DTD load). FundsXML needs none of those features. """ -import subprocess import sys -from pathlib import Path from lxml import etree -REPO_ROOT = Path(__file__).resolve().parents[2] - - -def ensure_schema(version: str) -> Path: - schema = REPO_ROOT / ".schema-cache" / version / "FundsXML.xsd" - if not schema.is_file(): - print(f"schema not cached; fetching official release {version}...", - file=sys.stderr) - subprocess.run([str(REPO_ROOT / "tools" / "fetch-schema.sh"), version], - check=True, stdout=subprocess.DEVNULL) - return schema +from fundsxml_schema import resolve_schema def main() -> int: @@ -37,7 +30,7 @@ def main() -> int: return 2 version, xml_path = sys.argv[1], sys.argv[2] - schema_path = ensure_schema(version) + schema_path = resolve_schema(version) # Hardened parser: no network, no entity resolution, no huge-tree blowups. safe = etree.XMLParser(no_network=True, resolve_entities=False, diff --git a/XSLT_DataQuality_Checks/README.md b/XSLT_DataQuality_Checks/README.md index 93eba5a..da018bc 100644 --- a/XSLT_DataQuality_Checks/README.md +++ b/XSLT_DataQuality_Checks/README.md @@ -196,11 +196,13 @@ implementation 'net.sf.saxon:Saxon-HE:12.4' ### Python -#### lxml (XSLT 1.0) +Install both engines once via the repo venv (cross-platform): ```bash -pip install lxml +python -m venv .venv && . .venv/bin/activate && pip install -e . ``` +#### lxml (XSLT 1.0) + ```python from lxml import etree @@ -214,9 +216,7 @@ with open("output.html", "wb") as f: ``` #### saxonche (XSLT 2.0/3.0) -```bash -pip install saxonche -``` +(installed by the `pip install -e .` above — `lxml` + `saxonche`) ```python from saxonche import PySaxonProcessor diff --git a/XSLT_Transformations/README.md b/XSLT_Transformations/README.md index b914fa9..2be7a45 100644 --- a/XSLT_Transformations/README.md +++ b/XSLT_Transformations/README.md @@ -19,7 +19,7 @@ Company-internal DQ rules live next door in | Stack | Entry point | Status | |-------|-------------|--------| | Java (s9api, no JAXB) | [`invocation/RunTransform.java`](invocation/RunTransform.java) | ✅ verified (via Maven Wrapper) | -| Python | [`invocation/run_transform.py`](invocation/run_transform.py) | needs `pip install saxonche` | +| Python | [`invocation/run_transform.py`](invocation/run_transform.py) | standalone via repo venv (`pip install -e .`) | | Node.js | see below | needs `npm i xslt3` (saxon-js) | The Java runner is standalone & cross-platform via the committed Maven Wrapper diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c1e9072 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,49 @@ +# Python build / dependency manifest for the FundsXML reference examples. +# +# WHY THIS EXISTS +# =============== +# The Python examples used to rely on ad-hoc `pip install lxml` / `pip install +# saxonche` and on tools/fetch-schema.sh (bash) to materialise the XSD. That is +# neither standalone nor cross-platform. This manifest makes the Python stack +# install with one idiomatic, OS-agnostic command and ships the in-language +# schema resolver as an importable module. +# +# STANDALONE BOOTSTRAP (any OS; venv works on Windows too) +# ======================================================= +# python -m venv .venv +# # Linux/macOS: . .venv/bin/activate Windows: .venv\Scripts\activate +# pip install -e . +# Then run any example with the venv's Python, e.g.: +# python XSD_Validation/python/validate.py 4.2.9 \ +# FundsXML_Files/4.2.9/positions/Mixed-Fund_Positions.xml +# +# `pip install -e .` exposes the top-level module `fundsxml_schema` +# (tools/fundsxml_schema.py): $FUNDSXML_SCHEMA_DIR -> .schema-cache -> official +# release download. No bash, no prior step, official release stays the source +# of truth (no committed catalog). + +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[project] +name = "fundsxml-examples" +version = "1.0.0" +description = "Standalone, cross-platform Python deps + schema resolver for the FundsXML reference examples" +requires-python = ">=3.9" +dependencies = [ + # XSD validation, iterparse streaming, DOM round-trips, the xml_equiv + # comparator, JSON data binding, large-file split/delta. + "lxml>=4.9", + # Saxon's Python API — the XSLT 2.0 / XQuery 3.1 / Schematron(xslt2) + # examples need a Saxon-class engine (lxml is XSLT 1.0 only). + "saxonche>=12.4", +] + +[tool.setuptools] +# Single-module distribution: expose tools/fundsxml_schema.py as the top-level +# importable `fundsxml_schema`. Nothing else in the repo is packaged. +py-modules = ["fundsxml_schema"] + +[tool.setuptools.package-dir] +"" = "tools" diff --git a/tools/fundsxml_schema.py b/tools/fundsxml_schema.py new file mode 100644 index 0000000..8f7144d --- /dev/null +++ b/tools/fundsxml_schema.py @@ -0,0 +1,105 @@ +"""Resolve the official FundsXML XSD for a given version — standalone & +cross-platform, no bash, no prior tool step. + +This is the Python counterpart of the in-language resolver used by the Java +example (XSD_Validation/java/XsdValidate.java). Same 3-step convention in every +stack: + + 1. ``$FUNDSXML_SCHEMA_DIR`` — a directory holding ``FundsXML.xsd`` (plus the + ``xmldsig-core-schema.xsd`` sibling for 4.2.9+). Used as-is, NO network. + The escape hatch for locked-down corporate networks / offline use. + 2. ``/.schema-cache//FundsXML.xsd`` — reused if present. + 3. download from the official GitHub release (following the 302), caching + into ``.schema-cache//``; the relative ``xmldsig-core-schema.xsd`` + sibling is fetched only when ``FundsXML.xsd`` actually imports it (4.2.9+). + +The source of truth stays the official release URL — no committed catalog. +``urllib`` follows the GitHub 302 to objects.githubusercontent.com on its own. + +Installed as a top-level module via the repo's ``pyproject.toml`` (``pip +install -e .``), so every Python example can ``from fundsxml_schema import +resolve_schema``. Also runnable directly:: + + python -m fundsxml_schema 4.2.9 # prints the resolved path +""" +from __future__ import annotations + +import os +import sys +import tempfile +import urllib.request +from pathlib import Path + +RELEASE_BASE = "https://github.com/fundsxml/schema/releases/download/" + +# tools/fundsxml_schema.py -> repo root is one level up. An editable install +# keeps __file__ pointing at this source file, so this stays correct. +_DEFAULT_REPO_ROOT = Path(__file__).resolve().parents[1] + + +def _download(url: str, out: Path) -> None: + """GET *url* (urllib follows the GitHub 302) atomically into *out*.""" + print(f"schema: fetch {url}", file=sys.stderr) + with urllib.request.urlopen(url, timeout=60) as resp: # noqa: S310 (trusted host) + if resp.status != 200: + raise RuntimeError(f"download failed (HTTP {resp.status}): {url}") + data = resp.read() + out.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=str(out.parent), suffix=".part") + try: + with os.fdopen(fd, "wb") as fh: + fh.write(data) + os.replace(tmp, out) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def resolve_schema(version: str, repo_root: Path | None = None) -> Path: + """Return a local path to ``FundsXML.xsd`` for *version* (see module doc).""" + # 1. Offline / corporate-network escape hatch: a hand-placed copy. + env_dir = os.environ.get("FUNDSXML_SCHEMA_DIR") + if env_dir: + xsd = Path(env_dir) / "FundsXML.xsd" + if not xsd.is_file(): + raise FileNotFoundError( + f"FUNDSXML_SCHEMA_DIR set but {xsd} not found") + print(f"schema: using $FUNDSXML_SCHEMA_DIR -> {xsd}", file=sys.stderr) + return xsd + + root = Path(repo_root) if repo_root else _DEFAULT_REPO_ROOT + cache_dir = root / ".schema-cache" / version + xsd = cache_dir / "FundsXML.xsd" + + # 2. Local cache (shared by every stack, gitignored). + if xsd.is_file(): + print(f"schema: cached -> {xsd}", file=sys.stderr) + return xsd + + # 3. Download from the official release (source of truth). + _download(f"{RELEASE_BASE}{version}/FundsXML.xsd", xsd) + # From 4.2.9 on, FundsXML.xsd imports xmldsig-core-schema.xsd via a + # relative path — it must sit next to FundsXML.xsd or the schema does not + # compile. Fetch the sibling only when it is actually referenced. + if "xmldsig-core-schema.xsd" in xsd.read_text(encoding="utf-8"): + _download(f"{RELEASE_BASE}{version}/xmldsig-core-schema.xsd", + cache_dir / "xmldsig-core-schema.xsd") + return xsd + + +def _main(argv: list[str]) -> int: + if not 1 <= len(argv) <= 2: + print("usage: python -m fundsxml_schema [repo-root]", + file=sys.stderr) + return 2 + version = argv[0] + root = Path(argv[1]) if len(argv) == 2 else None + print(resolve_schema(version, root)) # path on stdout (scriptable) + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main(sys.argv[1:]))