diff --git a/.github/workflows/inspect-r-api-update.yml b/.github/workflows/inspect-r-api-update.yml index b0e5da4c..dc0db3d4 100644 --- a/.github/workflows/inspect-r-api-update.yml +++ b/.github/workflows/inspect-r-api-update.yml @@ -100,6 +100,10 @@ jobs: id: live_parity continue-on-error: true run: | + # This job does not set up R; live-R verification is delegated to the + # parity-autofix workflow (dispatched below), which installs R. Here we + # gate the mapped parity tests against the committed cache, so pass + # --skip-install instead of trying to install R from local source. if [ "${{ steps.payload.outputs.fresh_cache }}" = "true" ]; then python scripts/run_live_r_parity_for_changed_api.py \ --plan sync/last_r_api_plan.json \ @@ -110,6 +114,7 @@ jobs: python scripts/run_live_r_parity_for_changed_api.py \ --plan sync/last_r_api_plan.json \ --r-checkout upstream/NNS \ + --skip-install \ --out sync/last_live_r_parity_report.md fi diff --git a/.github/workflows/parity-autofix.yml b/.github/workflows/parity-autofix.yml index 67bbd12a..004ad94f 100644 --- a/.github/workflows/parity-autofix.yml +++ b/.github/workflows/parity-autofix.yml @@ -112,18 +112,24 @@ jobs: id: r_install continue-on-error: true run: | - # install_local_r_nns.py installs from the vendored tools/NNS by - # default; the upstream checkout is the recorded truth for this commit. - python scripts/install_local_r_nns.py + # Install the recorded R code itself (upstream checkout at r_commit), + # not the vendored tools/NNS snapshot, so the fidelity test runs + # against the exact R behavior that triggered this workflow. + python scripts/install_local_r_nns.py \ + --source upstream/NNS \ + --expected-version "${{ steps.payload.outputs.r_version }}" - name: Run live R parity to reproduce divergence id: live_parity continue-on-error: true run: | + # --live recomputes the mapped parity subset from the freshly installed + # live R (committed cache moved aside, offline toggles cleared) and + # compares it against Python. A failure means real behavior diverged. python scripts/run_live_r_parity_for_changed_api.py \ --plan sync/last_r_api_plan.json \ --r-checkout upstream/NNS \ - --skip-install \ + --live \ --out sync/last_live_r_parity_report.md - name: Classify live parity outcome diff --git a/scripts/install_local_r_nns.py b/scripts/install_local_r_nns.py index b2cdb448..1a780f3e 100644 --- a/scripts/install_local_r_nns.py +++ b/scripts/install_local_r_nns.py @@ -17,6 +17,7 @@ from __future__ import annotations +import argparse import shutil import subprocess import sys @@ -34,8 +35,24 @@ ) -def _resolve_source() -> Path: - """Return the vendored NNS source path, preferring the extracted directory.""" +def _resolve_source(override: Path | None = None) -> Path: + """Return the NNS source path to install. + + With no override, prefer the vendored extracted directory and fall back to + the vendored tarball. With an override (for example an upstream checkout at a + recorded R commit), install that path directly after validating it is a + package source directory or tarball. + """ + + if override is not None: + if override.is_dir() and (override / "DESCRIPTION").is_file(): + return override + if override.is_file(): + return override + raise SystemExit( + f"ERROR: --source {override} is not an R package source. Expected a " + "directory containing DESCRIPTION or a package tarball." + ) if (_SOURCE_DIR / "DESCRIPTION").is_file(): return _SOURCE_DIR @@ -58,9 +75,32 @@ def _require(tool: str) -> str: def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--source", + type=Path, + default=None, + help=( + "Install from this R package source (directory with DESCRIPTION or a " + "tarball) instead of the vendored tools/NNS. Use an upstream checkout " + "to install live R NNS at a recorded commit." + ), + ) + parser.add_argument( + "--expected-version", + default=_EXPECTED_VERSION, + help=( + "Package version the install must report after loading. Defaults to " + f"{_EXPECTED_VERSION!r}. Pass the recorded upstream version when " + "installing from a non-vendored source." + ), + ) + args = parser.parse_args() + expected_version = args.expected_version or _EXPECTED_VERSION + r_bin = _require("R") rscript_bin = _require("Rscript") - source = _resolve_source() + source = _resolve_source(args.source) print(f"Installing R NNS from local source: {source} (not CRAN)") install = subprocess.run( @@ -86,15 +126,15 @@ def main() -> int: installed_version = probe.stdout.strip() print(f"Installed NNS version: {installed_version}") - if installed_version != _EXPECTED_VERSION: + if installed_version != expected_version: print( "ERROR: installed NNS version " - f"{installed_version!r} does not match expected {_EXPECTED_VERSION!r}.", + f"{installed_version!r} does not match expected {expected_version!r}.", file=sys.stderr, ) return 1 - print(f"OK: R NNS {_EXPECTED_VERSION} installed from local source.") + print(f"OK: R NNS {expected_version} installed from local source.") return 0 diff --git a/scripts/run_live_r_parity_for_changed_api.py b/scripts/run_live_r_parity_for_changed_api.py index e75a03a4..4abcf04f 100644 --- a/scripts/run_live_r_parity_for_changed_api.py +++ b/scripts/run_live_r_parity_for_changed_api.py @@ -3,12 +3,26 @@ import argparse import json import os +import shutil import subprocess import sys from pathlib import Path from typing import Any REPORT_DEFAULT = Path("sync/last_live_r_parity_report.md") +R_CACHE_PATH = Path("tests/_r_cache.json") +# Backup suffix is gitignored (see .gitignore) so it never lands in a PR. +R_CACHE_BACKUP = R_CACHE_PATH.with_suffix(".json.bak") + +# Toggles that force tests/_r.py into offline/cache-only mode. Clearing them lets +# a parity test actually shell out to live R on a cache miss. +OFFLINE_TOGGLES = ( + "CI", + "NNS_R_CACHE_ONLY", + "PYNNS_R_CACHE_ONLY", + "NNS_OFFLINE", + "PYNNS_OFFLINE", +) def load_json(path: Path) -> Any: @@ -21,8 +35,15 @@ def write_report(path: Path, lines: list[str]) -> None: print(path) -def run(cmd: list[str], env_extra: dict[str, str] | None = None) -> int: +def run( + cmd: list[str], + env_extra: dict[str, str] | None = None, + clear_offline: bool = False, +) -> int: env = os.environ.copy() + if clear_offline: + for name in OFFLINE_TOGGLES: + env.pop(name, None) if env_extra: env.update(env_extra) print("+ " + " ".join(cmd)) @@ -30,12 +51,102 @@ def run(cmd: list[str], env_extra: dict[str, str] | None = None) -> int: return completed.returncode +def run_live_subset(out: Path, header: list[str], parity_tests: list[str]) -> None: + """Recompute the mapped parity subset from live R and compare against Python. + + R NNS must already be installed (the workflow installs it from the upstream + checkout at the recorded commit). The committed cache is moved aside so every + mapped ``nns()`` call shells out to live R, the mapped tests run with the + offline toggles cleared, then the committed cache is restored unchanged. A + test failure means public Python behavior diverged from live R. + """ + + if shutil.which("Rscript") is None: + lines = [*header, + "## Result: live R unavailable", + "", + "`Rscript` is not on PATH, so live R parity could not be verified. " + "Install R NNS before running with `--live`.", + ] + write_report(out, lines) + raise SystemExit(4) + + existing_tests = [t for t in parity_tests if Path(t).exists()] + missing_tests = [t for t in parity_tests if not Path(t).exists()] + if not existing_tests: + lines = [*header, + "## Result: no mapped parity tests present", + "", + "No mapped parity test paths exist on disk; manual review is " + "recommended.", + ] + if missing_tests: + lines.extend(["", "Missing mapped tests:", ""]) + lines.extend(f"- `{t}`" for t in missing_tests) + write_report(out, lines) + return + + cmd = [sys.executable, "-m", "pytest", "-q", "-n", "0", *existing_tests] + moved = False + try: + if R_CACHE_PATH.exists(): + shutil.move(str(R_CACHE_PATH), str(R_CACHE_BACKUP)) + moved = True + # With the cache absent and offline toggles cleared, each mapped nns() + # call recomputes against the freshly installed live R and the test + # asserts the Python implementation matches it. + code = run(cmd, clear_offline=True) + finally: + if R_CACHE_PATH.exists(): + R_CACHE_PATH.unlink() + if moved: + shutil.move(str(R_CACHE_BACKUP), str(R_CACHE_PATH)) + + if code != 0: + lines = [*header, + "## Result: live R parity diverged", + "", + "Mapped parity tests recomputed every R value from the freshly " + "installed live R NNS and the Python implementation did not match. " + "Public Python behavior differs from live R at the recorded commit.", + "", + f"Failing command: `{' '.join(cmd)}`", + f"Exit status: `{code}`", + ] + write_report(out, lines) + raise SystemExit(code) + + lines = [*header, + "## Result: live R parity passed (recomputed from live R)", + "", + "Every mapped parity value was recomputed from the freshly installed " + "live R NNS and matched the Python implementation.", + "", + "Tests run:", + "", + ] + lines.extend(f"- `{t}`" for t in existing_tests) + if missing_tests: + lines.extend(["", "Skipped missing mapped tests:", ""]) + lines.extend(f"- `{t}`" for t in missing_tests) + write_report(out, lines) + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--plan", type=Path, default=Path("sync/last_r_api_plan.json")) parser.add_argument("--r-checkout", type=Path, required=False) parser.add_argument("--fresh-cache", action="store_true") parser.add_argument("--skip-install", action="store_true") + parser.add_argument( + "--live", + action="store_true", + help=( + "Recompute the mapped parity subset from already-installed live R " + "(toggles cleared, committed cache moved aside) and compare against " + "Python. A failure means Python diverged from live R." + ), + ) parser.add_argument("--out", type=Path, default=REPORT_DEFAULT) args = parser.parse_args() @@ -51,6 +162,7 @@ def main() -> None: f"- R checkout: `{args.r_checkout}`", f"- Fresh cache requested: `{args.fresh_cache}`", f"- Skip install: `{args.skip_install}`", + f"- Live R recompute: `{args.live}`", "", ] @@ -67,6 +179,13 @@ def main() -> None: write_report(args.out, lines) raise SystemExit(2) + # Live recompute mode: the true fidelity test. It recomputes the mapped + # subset from already-installed live R regardless of cache state, so it also + # covers version bumps without a separate fresh-cache pass. + if args.live: + run_live_subset(args.out, header, parity_tests) + return + # 3. Fresh cache required but not requested. if requires_fresh_cache and not args.fresh_cache: lines = [*header,