Bit-stable parity-cache inputs; shard the R cache per function; drop the numpy pin

.github/workflows/inspect-r-api-update.yml

@@ -72,11 +72,7 @@ jobs:
       - name: Install Python build and test tools
         run: |
           python -m pip install -U pip
-          # numpy pinned <2.5: the R parity cache keys hash test inputs generated via
-          # multivariate_normal (LAPACK SVD). numpy 2.5.x bundles an OpenBLAS whose SVD
-          # kernels differ on some runner CPUs, changing inputs bit-for-bit and causing
-          # cache misses. Re-evaluate at the next full live-R cache regeneration.
-          python -m pip install build scikit-build-core nanobind pytest ruff mypy "numpy<2.5" scipy
+          python -m pip install build scikit-build-core nanobind pytest ruff mypy numpy scipy
           python -m pip install hypothesis pytest-benchmark pytest-xdist
 
       - name: Plan R API parity review

.github/workflows/native-backend-ci.yml

@@ -25,11 +25,7 @@ jobs:
       - name: Install build tools
         run: |
           python -m pip install -U pip
-          # numpy pinned <2.5: the R parity cache keys hash test inputs generated via
-          # multivariate_normal (LAPACK SVD). numpy 2.5.x bundles an OpenBLAS whose SVD
-          # kernels differ on some runner CPUs, changing inputs bit-for-bit and causing
-          # cache misses. Re-evaluate at the next full live-R cache regeneration.
-          python -m pip install build scikit-build-core nanobind pytest ruff mypy "numpy<2.5" scipy
+          python -m pip install build scikit-build-core nanobind pytest ruff mypy numpy scipy
           python -m pip install hypothesis pytest-benchmark pytest-xdist
 
       - name: Install package editable

.github/workflows/parity-autofix.yml

@@ -86,11 +86,7 @@ jobs:
       - name: Install Python build and test tools
         run: |
           python -m pip install -U pip
-          # numpy pinned <2.5: the R parity cache keys hash test inputs generated via
-          # multivariate_normal (LAPACK SVD). numpy 2.5.x bundles an OpenBLAS whose SVD
-          # kernels differ on some runner CPUs, changing inputs bit-for-bit and causing
-          # cache misses. Re-evaluate at the next full live-R cache regeneration.
-          python -m pip install build scikit-build-core nanobind pytest ruff mypy "numpy<2.5" scipy
+          python -m pip install build scikit-build-core nanobind pytest ruff mypy numpy scipy
           python -m pip install hypothesis pytest-benchmark pytest-xdist
 
       - name: Install package editable
@@ -183,5 +179,5 @@ jobs:
             Python output, first divergent intermediate, and affected files.
             Native differences route through NNS-core. A DESCRIPTION/version or
             behavior change is a separate, reviewed cache regeneration. Do not
-            edit `tests/_r_cache.json` to force a pass.
+            edit the `tests/_r_cache/` shards to force a pass.
           commit-message: Parity review reports for R NNS ${{ steps.payload.outputs.r_commit }}

.github/workflows/sync-from-nns-core.yml

@@ -69,11 +69,7 @@ jobs:
       - name: Install build tools
         run: |
           python -m pip install -U pip
-          # numpy pinned <2.5: the R parity cache keys hash test inputs generated via
-          # multivariate_normal (LAPACK SVD). numpy 2.5.x bundles an OpenBLAS whose SVD
-          # kernels differ on some runner CPUs, changing inputs bit-for-bit and causing
-          # cache misses. Re-evaluate at the next full live-R cache regeneration.
-          python -m pip install build scikit-build-core nanobind pytest ruff mypy "numpy<2.5" scipy
+          python -m pip install build scikit-build-core nanobind pytest ruff mypy numpy scipy
           python -m pip install hypothesis pytest-benchmark pytest-xdist
 
       - name: Vendor NNS-core snapshot

.gitignore

@@ -16,7 +16,7 @@ __pycache__/
 
 # R cache lock and fresh-regeneration backup
 tests/_r_cache.lock
-tests/_r_cache.json.bak
+tests/_r_cache.bak/
 
 # Vendored R NNS local-install build artifacts
 tools/NNS/src/*.o

docs/parity.md

@@ -9,7 +9,7 @@ and remains the native C++ foundation for accelerated partial-moment routines.
 Full package parity is **not** claimed. Parity is bounded by the committed tests
 and cache:
 
-- `tests/_r_cache.json` — cache-only R result fixtures (2,406 keyed entries,
+- `tests/_r_cache/` — cache-only R result fixtures sharded per R function (one JSON each,
   schema version `1`, `nns_version == "13.0"`),
 - `tests/parity/` — public behavior parity checks,
 - `tests/invariants/` — Python-native contracts and invariants, and
@@ -85,7 +85,7 @@ If full regeneration is slow or unstable, regenerate deterministic chunks one
 file at a time, for example
 `python scripts/regenerate_r_cache.py -- -n 0 tests/parity/test_core.py`, then
 continue through the remaining parity files. The committed result must remain a
-single valid `tests/_r_cache.json` with `nns_version == "13.0"`,
+valid `tests/_r_cache/` shards with `nns_version == "13.0"`,
 `schema_version == 1`, and non-empty `entries`; `scripts/regenerate_r_cache.py`
 enforces those guardrails after the pytest run.
 
@@ -130,7 +130,7 @@ NNS R API change
 ### Fix rules (applied by the maintainer; later by the agent)
 
 - Edit **`src/nns/**` only**. Never edit `extern/NNS-core/**`, `tools/NNS/**`,
-  or `tests/_r_cache.json` to make a check pass.
+  or the `tests/_r_cache/` shards to make a check pass.
 - Classify the root cause and act accordingly:
   - **Python port bug** → fix in `src/nns/**`.
   - **R changed behavior** → do not chase a cache value; cache regeneration is a

docs/plot_parity_policy.md

@@ -36,7 +36,7 @@ colors and which element they sit on*, never rendered images.
 
 - Numeric return values (scalars, vectors, matrices, nested result dicts) from
   every ported function, against committed R fixtures and the committed R cache
-  (`tests/_r_cache.json`).
+  (`tests/_r_cache/`).
 - Structural contracts (result keys, shapes, dtypes, finiteness) via the
   invariant suite.
 

scripts/regenerate_r_cache.py

@@ -2,15 +2,15 @@
 """Regenerate committed R parity cache entries with a local R/NNS install.
 
 CI should not run this script. It intentionally clears cache-only/offline toggles
-and invokes pytest so tests/_r.py can refresh tests/_r_cache.json as needed.
+and invokes pytest so tests/_r.py can refresh the tests/_r_cache/ shards as needed.
 
 Usage::
 
     python scripts/regenerate_r_cache.py [--fresh] [-- PYTEST_ARGS...]
 
 By default, existing cache entries are reused and only cache misses call R.
-With ``--fresh``, the existing ``tests/_r_cache.json`` is moved aside to
-``tests/_r_cache.json.bak`` and regeneration starts from an empty cache, so
+With ``--fresh``, the existing ``tests/_r_cache/`` directory is moved aside to
+``tests/_r_cache.bak/`` and regeneration starts from an empty cache, so
 every entry is produced by a live R call. ``--fresh`` refuses to run in CI and
 requires a working local R NNS install reporting ``packageVersion("NNS")`` of
 13.0 (see ``scripts/install_local_r_nns.py``).
@@ -27,10 +27,10 @@
 from pathlib import Path
 from typing import Any
 
-_CACHE_PATH = Path(__file__).resolve().parents[1] / "tests" / "_r_cache.json"
-_CACHE_BACKUP_PATH = _CACHE_PATH.with_suffix(".json.bak")
+_CACHE_DIR = Path(__file__).resolve().parents[1] / "tests" / "_r_cache"
+_CACHE_BACKUP_DIR = _CACHE_DIR.with_name("_r_cache.bak")
 _NNS_VERSION = "13.0"
-_SCHEMA_VERSION = 1
+_SCHEMA_VERSION = 2
 
 _OFFLINE_TOGGLES = (
     "PYNNS_R_CACHE_ONLY",
@@ -42,58 +42,63 @@
 
 
 def _validate_cache() -> int:
-    if not _CACHE_PATH.exists():
-        print(f"ERROR: R cache validation failed: {_CACHE_PATH} does not exist.", file=sys.stderr)
+    if not _CACHE_DIR.is_dir():
+        print(f"ERROR: R cache validation failed: {_CACHE_DIR} does not exist.", file=sys.stderr)
         return 1
-    if _CACHE_PATH.stat().st_size == 0:
-        print(f"ERROR: R cache validation failed: {_CACHE_PATH} is empty.", file=sys.stderr)
-        return 1
-
-    try:
-        cache: Any = json.loads(_CACHE_PATH.read_text(encoding="utf-8"))
-    except json.JSONDecodeError as exc:
-        print(
-            f"ERROR: R cache validation failed: {_CACHE_PATH} is not valid JSON: {exc}.",
-            file=sys.stderr,
-        )
-        return 1
-
-    if not isinstance(cache, dict):
-        print(
-            f"ERROR: R cache validation failed: {_CACHE_PATH} top-level value is not an object.",
-            file=sys.stderr,
-        )
-        return 1
-    if cache.get("nns_version") != _NNS_VERSION:
+    shard_paths = sorted(_CACHE_DIR.glob("*.json"))
+    if not shard_paths:
         print(
-            "ERROR: R cache validation failed: "
-            f"expected nns_version {_NNS_VERSION!r}, got {cache.get('nns_version')!r}.",
-            file=sys.stderr,
-        )
-        return 1
-    if cache.get("schema_version") != _SCHEMA_VERSION:
-        print(
-            "ERROR: R cache validation failed: "
-            f"expected schema_version {_SCHEMA_VERSION!r}, got {cache.get('schema_version')!r}.",
+            f"ERROR: R cache validation failed: {_CACHE_DIR} contains no shards.",
             file=sys.stderr,
         )
         return 1
 
-    entries = cache.get("entries")
-    if not isinstance(entries, dict):
-        print(
-            f"ERROR: R cache validation failed: {_CACHE_PATH} entries value is not an object.",
-            file=sys.stderr,
-        )
-        return 1
-    if not entries:
-        print(
-            f"ERROR: R cache validation failed: {_CACHE_PATH} entries object is empty.",
-            file=sys.stderr,
-        )
-        return 1
+    total_entries = 0
+    for shard_path in shard_paths:
+        try:
+            shard: Any = json.loads(shard_path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError as exc:
+            print(
+                f"ERROR: R cache validation failed: {shard_path} is not valid JSON: {exc}.",
+                file=sys.stderr,
+            )
+            return 1
+        if not isinstance(shard, dict):
+            print(
+                f"ERROR: R cache validation failed: {shard_path} top-level value "
+                "is not an object.",
+                file=sys.stderr,
+            )
+            return 1
+        if shard.get("nns_version") != _NNS_VERSION:
+            print(
+                f"ERROR: R cache validation failed: {shard_path} "
+                f"expected nns_version {_NNS_VERSION!r}, got {shard.get('nns_version')!r}.",
+                file=sys.stderr,
+            )
+            return 1
+        if shard.get("schema_version") != _SCHEMA_VERSION:
+            print(
+                f"ERROR: R cache validation failed: {shard_path} "
+                f"expected schema_version {_SCHEMA_VERSION!r}, got "
+                f"{shard.get('schema_version')!r}.",
+                file=sys.stderr,
+            )
+            return 1
+        entries = shard.get("entries")
+        if not isinstance(entries, dict) or not entries:
+            print(
+                f"ERROR: R cache validation failed: {shard_path} entries object "
+                "is missing or empty.",
+                file=sys.stderr,
+            )
+            return 1
+        total_entries += len(entries)
 
-    print(f"OK: {_CACHE_PATH} contains {len(entries)} entries for NNS {_NNS_VERSION}.")
+    print(
+        f"OK: {_CACHE_DIR} contains {total_entries} entries across "
+        f"{len(shard_paths)} shards for NNS {_NNS_VERSION}."
+    )
     return 0
 
 
@@ -149,7 +154,7 @@ def main() -> int:
         "--fresh",
         action="store_true",
         help=(
-            "Move the existing cache to tests/_r_cache.json.bak and regenerate every "
+            "Move the existing cache to tests/_r_cache.bak/ and regenerate every "
             "entry from a live R call. Refuses to run in CI."
         ),
     )
@@ -171,9 +176,11 @@ def main() -> int:
         verify_status = _verify_live_r_nns()
         if verify_status:
             return verify_status
-        if _CACHE_PATH.exists():
-            _CACHE_PATH.replace(_CACHE_BACKUP_PATH)
-            print(f"Moved existing cache to {_CACHE_BACKUP_PATH}; starting from empty cache.")
+        if _CACHE_DIR.is_dir():
+            if _CACHE_BACKUP_DIR.is_dir():
+                shutil.rmtree(_CACHE_BACKUP_DIR)
+            _CACHE_DIR.replace(_CACHE_BACKUP_DIR)
+            print(f"Moved existing cache to {_CACHE_BACKUP_DIR}; starting from empty cache.")
         else:
             print("No existing cache found; starting from empty cache.")
 

scripts/run_live_r_parity_for_changed_api.py

@@ -10,9 +10,9 @@
 from typing import Any
 
 REPORT_DEFAULT = Path("sync/last_live_r_parity_report.md")
-R_CACHE_PATH = Path("tests/_r_cache.json")
+R_CACHE_PATH = Path("tests/_r_cache")
 # Backup suffix is gitignored (see .gitignore) so it never lands in a PR.
-R_CACHE_BACKUP = R_CACHE_PATH.with_suffix(".json.bak")
+R_CACHE_BACKUP = R_CACHE_PATH.with_name("_r_cache.bak")
 
 # Toggles that force tests/_r.py into offline/cache-only mode. Clearing them lets
 # a parity test actually shell out to live R on a cache miss.
@@ -90,6 +90,8 @@ def run_live_subset(out: Path, header: list[str], parity_tests: list[str]) -> No
     moved = False
     try:
         if R_CACHE_PATH.exists():
+            if R_CACHE_BACKUP.exists():
+                shutil.rmtree(R_CACHE_BACKUP)
             shutil.move(str(R_CACHE_PATH), str(R_CACHE_BACKUP))
             moved = True
         # With the cache absent and offline toggles cleared, each mapped nns()
@@ -98,7 +100,7 @@ def run_live_subset(out: Path, header: list[str], parity_tests: list[str]) -> No
         code = run(cmd, clear_offline=True)
     finally:
         if R_CACHE_PATH.exists():
-            R_CACHE_PATH.unlink()
+            shutil.rmtree(R_CACHE_PATH)
         if moved:
             shutil.move(str(R_CACHE_BACKUP), str(R_CACHE_PATH))
 

scripts/sync_nns_core_snapshot.py

@@ -60,7 +60,7 @@ def main() -> None:
         "vendored_core_path": "extern/NNS-core",
         "vendored_r_path": "tools/NNS",
         "vendored_r_tarball": f"tools/NNS_{args.r_version}.tar.gz",
-        "r_cache_path": "tests/_r_cache.json",
+        "r_cache_path": "tests/_r_cache",
         "notes": (
             "NNS-python consumes accepted NNS-core snapshots for native code and "
             "verifies public Python behavior against live or cached R NNS."

scripts/sync_r_nns_snapshot.py

@@ -93,7 +93,7 @@ def main() -> None:
             "python_commit": None,
             "vendored_r_path": "tools/NNS",
             "vendored_r_tarball": str(tarball),
-            "r_cache_path": "tests/_r_cache.json",
+            "r_cache_path": "tests/_r_cache",
         }
     )
     manifest.setdefault("core_repo", "OVVO-Financial/NNS-core")

sync/nns_source.json

@@ -10,6 +10,6 @@
   "vendored_core_path": "extern/NNS-core",
   "vendored_r_path": "tools/NNS",
   "vendored_r_tarball": "tools/NNS_13.0.tar.gz",
-  "r_cache_path": "tests/_r_cache.json",
+  "r_cache_path": "tests/_r_cache",
   "notes": "NNS-python consumes accepted NNS-core snapshots for native code and verifies public Python behavior against live or cached R NNS."
 }

sync/r_api_map.json

@@ -57,7 +57,7 @@
     "python_modules": [
       "pyproject.toml",
       "tools/NNS",
-      "tests/_r_cache.json"
+      "tests/_r_cache"
     ],
     "parity_tests": ["tests/parity"],
     "requires_fresh_cache": true