diff --git a/.claude/teams/audit/v1.1/EVIDENCE/executor.md b/.claude/teams/audit/v1.1/EVIDENCE/executor.md new file mode 100644 index 0000000..2353777 --- /dev/null +++ b/.claude/teams/audit/v1.1/EVIDENCE/executor.md @@ -0,0 +1,111 @@ +# Executor — close.py collision group (T-01, T-02, T-10) + +Branch: release/v1.0 +Owner: engineering-executor +Mutmut baseline at start: 169 killed mutants +Pre-existing test count at start: 225 collected (224 passed + 1 skipped) + +## Task T-01 (Lazy torch import) + +### What I did +Replaced the module-level `try: import torch as _torch / _has_torch` block in +`src/gpucheck/assertions/close.py` with a lazy `_torch_mod()` helper that +caches the imported module (or `None`) using a sentinel. Both call sites +inside `assert_close` (device-type detection and the GPU fast-path) now bind +`_torch = _torch_mod()` once at the top of the function and guard with +`_torch is not None`. + +### Files modified +- `src/gpucheck/assertions/close.py`: lazy-import refactor; `_has_torch` removed; `_torch_mod()` is the single point of access. + +### Files created +- `DIFF_LOG.md` (this iteration logs the T-01 change). +- `.claude/teams/audit/v1.1/EVIDENCE/executor.md` (this file). + +### Design decisions made during implementation +- Used a sentinel (`_TORCH_UNRESOLVED = object()`) instead of `None` for the + cache initial state, so a process where torch is genuinely absent still + short-circuits after the first probe (caches `None` permanently). +- Kept the inner `import torch` inside the `__cuda_array_interface__` + fallback as-is rather than routing it through `_torch_mod()`. That branch + needed torch *and* dlpack to be present and was already lazy by virtue of + living inside a function body — re-routing through the helper would have + changed behaviour (it would no longer except `ImportError` locally). + +### Potential blast radius +- If `gpucheck.assertions.close` is imported on a torch-less host, the + module load path no longer raises or warns. Behaviour matches the + previous code (which set `_has_torch = False` silently). +- The `_torch_cached` global is process-wide. If a test fixture + monkey-patches `sys.modules['torch']` after `_torch_mod()` has run once, + the cached value will be stale. Existing tests do not do this, but the + verifier should confirm. + +## Task T-02 (`.contiguous()` on slow path) + +### What I did +Added `.contiguous()` to both torch.Tensor branches of `_to_numpy` (the +primary `hasattr(tensor, "detach")` branch and the dlpack fallback in the +`__cuda_array_interface__` block). Wrote a new parametrized test module +`tests/test_assert_close_contiguous.py` covering three stride patterns: +slice (`[:, ::2]`), transpose (`.t()`), and broadcast (`.expand`). + +### Files modified +- `src/gpucheck/assertions/close.py`: two `.cpu().contiguous()` insertions; PM-4 citation comment. + +### Files created +- `tests/test_assert_close_contiguous.py`: 6 parametrized cases (3 stride patterns × 2 entry points). + +### Design decisions made during implementation +- Applied `.contiguous()` to the dlpack fallback as well even though only + the primary branch is in the strict T-02 scope. The same RuntimeError + surface exists in both code paths and the cost is negligible. Noted + here as an opportunistic widen so the reviewer can flag if undesired. +- Used `pytest.importorskip("torch")` rather than the existing + `_has_torch`-style guard so the module skips cleanly on torch-less + hosts (matches the lazy-import discipline from T-01). + +### Potential blast radius +- `.contiguous()` allocates a new tensor when the input is non-contiguous. + For very large stride-fuzzed tensors this could double peak memory in + the slow path. Existing CUDA fast-path (which bypasses `_to_numpy`) + already handles same-shape tensors without copying, so the regression is + bounded to mismatched / failing comparisons. + +## Task T-10 (Pin numeric fields in mismatch report) + +### What I did +Added `TestMismatchReportPinnedNumerics` to `tests/test_assertions.py` with +three new tests. Each test constructs simple integer-valued numpy inputs so +expected values can be hand-computed exactly, then asserts those values +appear verbatim (with `:.6e` formatter) in the rendered Rich report. Did +NOT modify `src/gpucheck/assertions/reporting.py` per task instructions. + +### Files modified +- `tests/test_assertions.py`: appended 3 tests targeting the ~30 surviving + reporting.py mutants from `EVIDENCE/mutator-survivors.md`. + +### Files created +- (none) + +### Design decisions made during implementation +- Used 2-D input in test 2 specifically so `np.unravel_index` is exercised + meaningfully (with a 1-D input, any axis-mutation would be a no-op). +- For the histogram count assertion, stripped ANSI escape codes via + `re.sub(r"\x1b\[[0-9;]*m", "", report)` before scanning digits because + Rich's coloured output otherwise leaks digits like `33`/`31` from + ``\x1b[33m`` and pollutes the digit-only filter. +- Selected unique-maximum diff values (4.5, 5.0) so the location index + is unambiguous — eliminating spurious passes if `nanargmax` is mutated + to e.g. `nanargmin` and the answer happens to coincide. + +### Potential blast radius +- The histogram-count test depends on the bar-rendering loop emitting + the count after the bar (`f" {bucket:>22s} | {bar} {count}"`). If the + format string is reordered (count before bar), the test would still + isolate the digits via the line-tail extraction, but the failure + message would be misleading. Acceptable trade-off for now. +- `5 / 6 (83.33%)` substring is whitespace-sensitive: if the table + formatter switches columns or pads differently, the test could + false-fail. Task scope is to test current behaviour; if reporting is + refactored, these tests must be updated. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index edfe733..690ed9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,13 @@ on: pull_request: branches: [main] +# Security finding CFG-2 (FINDINGS.md): default GITHUB_TOKEN scope is +# write-all. Explicitly restrict to read-only at the workflow level so +# nothing in this CI job can accidentally publish artifacts or comments +# unless a step elevates locally. +permissions: + contents: read + jobs: lint: runs-on: ubuntu-latest @@ -14,12 +21,22 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Install dependencies - run: pip install -e ".[dev]" + - name: Install uv + uses: astral-sh/setup-uv@v3 + - name: Install dependencies (locked) + # Use the committed uv.lock for reproducibility and supply-chain + # protection (security finding DEP-1). + run: | + uv venv + if [ -f uv.lock ]; then + uv sync --frozen --extra dev + else + uv pip install -e ".[dev]" + fi - name: Ruff check - run: ruff check src/ tests/ + run: uv run ruff check src/ tests/ - name: Mypy - run: mypy src/ + run: uv run mypy src/ test: runs-on: ubuntu-latest @@ -32,7 +49,15 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: pip install -e ".[dev]" + - name: Install uv + uses: astral-sh/setup-uv@v3 + - name: Install dependencies (locked) + run: | + uv venv + if [ -f uv.lock ]; then + uv sync --frozen --extra dev + else + uv pip install -e ".[dev]" + fi - name: Run tests - run: pytest --tb=short -q + run: uv run pytest --tb=short -q diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6daf575 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,269 @@ +# Changelog + +All notable changes to **gpucheck** will be documented in this file. + +The format is based on +[Keep a Changelog 1.1.0](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +--- + +## [Unreleased] + +### Added + +- **`@requires_arch` decorator (T-21).** Plural-form alias of + `@require_arch`, matching the existing `@requires_determinism` + spelling. Importable from `gpucheck.arch` (or + `gpucheck.arch.compatibility`). The new name is the canonical form + going forward. + +### Changed + +- *(reserved for post-1.0 work)* + +### Deprecated + +- **`@require_arch` (singular) is deprecated.** It now emits a + `DeprecationWarning` on use and will be removed in v1.2. Migrate + existing call sites to `@requires_arch` (plural). The deprecation + exists to make decorator naming consistent with + `@requires_determinism`; both decorators can now be searched as + `requires_*` in your codebase. + +### Fixed + +- *(reserved for post-1.0 work)* + +--- + +## [1.0.0rc1] — 2026-05-01 + +First release candidate of the v1.0 line. Four parallel engineering tracks +delivered: an Apple Silicon Metal Performance Shaders (MPS) backend, stride +and contiguity fuzzing, a thread-safe tolerance override stack, and a +release-bundle hardening pass. + +> **Headline:** gpucheck is no longer CUDA-only. The same fuzz playbook that +> found `triton#9838` (83% layer-norm error, OPEN) and `triton#9839` (FP16 +> matmul drift, CLOSED) now runs on Apple Silicon, with a curated 12-entry +> xfail list covering known-broken PyTorch MPS kernels. + +### Added + +- **MPS backend (Track A — `feat/track-a-mps`).** + - New `gpucheck.backends` package introduces a `runtime_checkable` + `Backend` Protocol (`src/gpucheck/backends/_protocol.py`) plus + `CUDABackend` (`backends/cuda.py`) and `MPSBackend` (`backends/mps.py`). + Public entry points: `gpucheck.available_backends()` and + `gpucheck.get_backend(name)`. + - `assert_close`, `gpu_benchmark`, `memory_tracker`, `gpu_device`, + `@devices`, and `@parametrize_gpu` now recognize `device="mps"`. + `@devices()` auto-detects MPS via `torch.backends.mps.is_available()`, + and `@devices("all")` includes MPS when present. + - **Deadlock-safe MPS benchmarking.** Per + [pytorch#162872](https://github.com/pytorch/pytorch/issues/162872), + the pattern `start.record(); end.record(); end.synchronize(); + start.elapsed_time(end)` deadlocks on MPS. `MPSBackend.event_timer` + routes to device-level `torch.mps.synchronize()` instead of + per-event `Event.synchronize()`. A test in `tests/test_backends.py` + AST-introspects the MPS path to assert no `Event.synchronize` call + site exists. + - **Per-kernel xfail registry.** A new `[tool.gpucheck.mps.xfail]` + section in `pyproject.toml` ships with **12 curated entries** drawn + from the research SYNTHESIS top-impact open MPS bugs. Loaded at + `pytest_configure` time; queryable via `gpucheck.is_mps_xfailed(op)`, + `gpucheck.mps_xfail_list()`, and `gpucheck.register_mps_xfail(op)`. + The 12 entries cite concrete upstream issues (pytorch#179352, + #179294, #173525, #175189, #142836, #174269, #181936, #96602, + #175190, #176296, #137001, #177116) — see + `.claude/teams/research/v1.0/SYNTHESIS.md` §Sub-Q 2 / §Sub-Q 7. + - **MPS tolerance overlay.** `_MPS_TOLERANCE_MULTIPLIERS` in + `src/gpucheck/assertions/tolerances.py` adds a 2× per-dtype multiplier + on top of the CUDA-calibrated baseline for FP32, FP16, and BF16. + The 2× multiplier is **PROVISIONAL** per SYNTHESIS §Sub-Q 7 — it is + a hypothesis grounded in precision-floor + Apple-no-FP16-tensor-cores + arguments, pending P99 calibration on Akash's M-machine. If observed + drift exceeds 2×, the affected op moves to the xfail registry rather + than further inflating tolerances. + - New install extras: `pip install gpucheck[mps]` and + `pip install gpucheck[apple]` (alias). Both pin `torch>=2.6` (the + floor where `torch.mps.synchronize()` is stable). +- **Stride / contiguity fuzzing (Track B — `feat/track-b-strides`).** + - New `src/gpucheck/fuzzing/strides.py` with a 7-category deterministic + corpus (`row-major`, `column-major`, `broadcast-induced`, `transpose`, + `slice`, `contiguous-after-clone`, `gather-induced`). + - Public surface: `fuzz_strides()`, `fuzz_strides_for_category()`, + `StrideStrategy` (Hypothesis), and `STRIDE_CATEGORIES`. + - `parametrize_gpu(stride_categories=...)` wiring threads stride + fuzzing through the existing dtype/shape/device matrix. +- **Thread-safe tolerance overrides (Track C — `feat/track-c-thread-safety`).** + - The override stack at `src/gpucheck/assertions/tolerances.py` is now + a `contextvars.ContextVar`. `tolerance_context(atol, rtol)` is safe + from `pytest-xdist` workers AND concurrent threads inside a single + worker. `asyncio` tasks are isolated per the standard `ContextVar` + semantics. **No user-facing API change.** +- **Release bundle (Track D — `feat/track-d-bundle`).** + - **HTML dashboard** — `gpucheck.reporting.HTMLReporter` produces a + self-contained static HTML artifact (`src/gpucheck/reporting/html.py`). + - **Determinism sanitizer** — `assert_deterministic`, + `@requires_determinism`, and `DeterminismError` + (`src/gpucheck/sanitizers/determinism.py`). Implements the + "fix seeds + run twice + compare" recipe required for MPS, where + bit-exact reproducibility is not guaranteed (SYNTHESIS §Sub-Q 4). + - **`uv.lock` committed** (DEP-1 mitigation). Downstream installs that + consume the lock benefit from supply-chain reproducibility. + - **CI hardened** — `.github/workflows/ci.yml` gains + `permissions: contents: read` (CFG-2) and installs from the locked + environment via `uv sync --frozen`. +- `MIGRATION.md`, `CONTRIBUTING.md`, and `CHANGELOG.md` published for the + first time. + +### Changed + +- **`GPUInfo` shape** (`src/gpucheck/arch/detection.py`) gains a + `backend: str = "cuda"` field. Existing CUDA-shaped fields + (`compute_capability`, `tensor_core_generation`, `cuda_version`) + remain in place; on MPS, they take backend-appropriate sentinel values. + See `MIGRATION.md` §1. +- **`assert_close` GPU fast-path** (`src/gpucheck/assertions/close.py`) + widened from CUDA-only to recognize both `cuda` and `mps` device types. + `compute_tolerance(...)` now accepts `device_type=` for MPS overlay routing. +- **Tolerance computation** (`src/gpucheck/assertions/tolerances.py`) + threads `device_type` through and applies `_MPS_TOLERANCE_MULTIPLIERS` + when `device_type == "mps"` (see PROVISIONAL note above). +- **`@devices(...)` semantics.** With no arguments, auto-detect now + includes `mps` on Apple Silicon hosts. The string `"all"` resolves to + the union of all detected devices (CUDA + MPS). +- **`gpu_benchmark` fixture** (`src/gpucheck/fixtures/benchmark.py`) now + branches on `device.type`: `_run_cuda` keeps the existing CUDA-event + path; `_run_mps` uses `torch.mps.event.Event` with device-level sync. +- **`pytest_configure` hook** (`src/gpucheck/plugin.py`) reads + `[tool.gpucheck.mps.xfail]` and `[tool.gpucheck.tolerances]` from + `pyproject.toml` via `tomllib` and applies them to the active session. +- **CI** runs `uv sync --frozen` instead of `pip install -e ".[dev]"`. +- **Reporting test coverage**: 0% → 98% (tracks console, JSON, CI + annotations, JUnit XML, PR comments, HTML dashboard). +- **Test count**: 117 → 224 (107 net new tests across the four tracks). + +### Deprecated + +- *(none in this release candidate)* + +### Removed + +- *(none — v1.0 is API-additive vs v0.1.0; see `MIGRATION.md` for shape + changes that did not require removal)* + +### Fixed + +- **Reporting module zero coverage** (was a documented gap in + `CLAUDE.md`'s "Known Weaknesses & Gaps") — closed by Track D. +- **Thread-safety in tolerance overrides** (was the `# NOT thread-safe` + comment at `assertions/tolerances.py`) — closed by Track C. +- **Stride/contiguity fuzzing gap** (was a documented gap) — closed by + Track B. +- **HTML/dashboard reporting gap** (was a documented gap) — closed by + Track D. +- **Determinism testing gap** (was a documented gap) — closed by + Track D's `sanitizers/determinism.py`. +- README "8 bugs" claim reconciled with the visible "Bugs found" table: + the README now states **8 bugs found via 511 test configurations, of + which 2 (`triton#9838` OPEN, `triton#9839` CLOSED) are externally + filed and verified upstream**. The remaining 5 in the table are + internal-ledger findings reproducible from `examples/`. See AUDIT.md + §A.5 item 32 and SYNTHESIS §Sub-Q 8. + +### Security + +- **CFG-2 (MEDIUM)** — `.github/workflows/ci.yml` now declares + `permissions: contents: read`, locking down the default + `GITHUB_TOKEN` write-all surface. (Track D.) +- **TM-E1 (MEDIUM)** — `src/gpucheck/sanitizers/race.py` validates + `CUDA_HOME` / `CUDA_PATH` against an allowlist of canonical install + prefixes (`/usr/local/cuda`, `/opt/nvidia/cuda`, `/opt/cuda`) using + `os.path.realpath`, with a warning on rejection. (Track C.) +- **DEP-1 (MEDIUM)** — `uv.lock` is now committed; CI installs from the + lockfile (`uv sync --frozen`). (Track D.) +- **N1, N2, N3, N5 (MPS design-stage)** — explicitly waived in + engineering CHARTER.md §"Explicit waivers": gpucheck v1.0 does not + shell out to `xcrun metal*`, does not read `task_info` directly, and + does not ship an MPS dispatch sanitizer. PyTorch's hardening is the + trust boundary. +- **N4 (MPS design-stage)** — partially mitigated by the `torch>=2.6` + floor on the `[mps]` extra and the DEP-1 lockfile; no third-party + Apple-only packages introduced. +- See `.claude/teams/security/v1.0/FINDINGS.md` for the full ledger + (0 CRITICAL · 0 HIGH · 3 MEDIUM · 13 LOW; **ADVISORY** verdict). + +### Migration + +- See [`MIGRATION.md`](./MIGRATION.md) for the v0.1.0 → v1.0 guide. +- v1.0 is API-additive: existing CUDA-only code continues to work. + The new entry points are `gpucheck.backends.get_backend()`, + `@devices("mps")`, the `[tool.gpucheck.mps.xfail]` config block, and + the `[mps]` install extra. +- The PROVISIONAL 2× MPS tolerance multiplier may be revised in + v1.0.0 final after M-machine calibration; users hard-coding overlays + should track this CHANGELOG. + +### Known issues + +- AMD ROCm and Intel XPU still unsupported; planned for a later release. + CUDA + MPS only. +- Multi-GPU NCCL communication testing, CUDA graph testing, and + gradient/backward testing remain out of scope. +- The 2× MPS tolerance multiplier is PROVISIONAL pending M-machine + P99 calibration; if measured drift exceeds 2× for a specific kernel, + that kernel will move to the xfail registry rather than further + inflating tolerances. + +--- + +## [0.1.0] — 2026-03 + +Initial PyPI release. + +### Added + +- `assert_close()` — dtype-aware tensor comparison with `k_dim` scaling, + `baseline_2x` mode, mixed-precision auto-resolution, and Rich-formatted + mismatch reports with error histograms. +- `@dtypes`, `@shapes`, `@devices`, `@parametrize_gpu` parametrize + decorators with predefined groups (`FLOAT_DTYPES`, `HALF_DTYPES`, + `ALL_DTYPES`, `FP8_DTYPES`, `SMALL_SHAPES`, `MEDIUM_SHAPES`, + `LARGE_SHAPES`, `EDGE_SHAPES`). +- `gpu_benchmark` fixture using CUDA events, L2 flushing, and IQR + outlier removal. +- `gpu_device` fixture and `GPUDevice` dataclass. +- `memory_tracker` fixture and `memory_guard` context manager. +- `fuzz_shapes()` deterministic corpus + `ShapeStrategy` Hypothesis + factory; `gpu_shapes()` and `gpu_tensors()` strategies; `random_inputs`, + `edge_inputs`, `mixed_inputs` generators. +- `check_memory_leaks()` and `run_with_sanitizer()` (NVIDIA + compute-sanitizer wrapper). +- `arch.detection.detect_gpus()` (pynvml-first, torch fallback) plus + `GPUInfo`, `@require_arch`, `@require_capability`. +- `analysis.regression.detect_regression()` (Mann-Whitney U + + Cohen's d + simplified E-Divisive); `compute_roofline`, + `classify_bottleneck`, `auto_classify_bottleneck`, + `render_roofline_ascii`. +- `reporting.console.ConsoleReporter`, `reporting.json.JSONReporter`, + GitHub Actions annotations, JUnit XML, PR comment generation. +- pytest hooks: `--gpu-device`, `--gpu-benchmark-warmup`, + `--gpu-benchmark-rounds`; markers `gpu`, `slow`, `multi_gpu`. + +### Findings (validated on NVIDIA GeForce GTX 1650, Turing SM75) + +- 8 bugs surfaced via 511 test configurations against Triton tutorials + and PyTorch CUDA ops. Externally filed and verified: + - [`triton#9838`](https://github.com/triton-lang/triton/issues/9838) + — 83% relative error in Triton tutorial layer-norm at `n_cols=17` + (OPEN as of 2026-05). + - [`triton#9839`](https://github.com/triton-lang/triton/issues/9839) + — FP16 index wrapping in Triton tutorial matmul, 0.125 abs error at + K=8192 (CLOSED). + +[Unreleased]: https://github.com/Akasxh/gpucheck/compare/v1.0.0rc1...HEAD +[1.0.0rc1]: https://github.com/Akasxh/gpucheck/releases/tag/v1.0.0rc1 +[0.1.0]: https://github.com/Akasxh/gpucheck/releases/tag/v0.1.0 diff --git a/CLAUDE.md b/CLAUDE.md index a0967ea..c1ef9cf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,23 +5,25 @@ gpucheck is a pytest plugin for GPU kernel testing. It provides dtype-aware asse **Author:** Akash (drakathakash@gmail.com) **License:** Apache-2.0 -**PyPI:** gpucheck v0.1.0 +**PyPI:** gpucheck v1.0.0rc1 (initial PyPI publish was v0.1.0) **Python:** >=3.10 +**Backends:** CUDA + Apple MPS (v1.0) ## Architecture ``` src/gpucheck/ __init__.py # Lazy public API (import-time zero-cost) - plugin.py # pytest hooks: markers, fixtures, terminal summary - assertions/ # assert_close(), tolerances, Rich mismatch reports - decorators/ # @dtypes, @shapes, @devices, @parametrize_gpu - fixtures/ # gpu_benchmark (CUDA events), memory_tracker, gpu_device - fuzzing/ # fuzz_shapes(), ShapeStrategy, edge_inputs(), gpu_tensors() - sanitizers/ # memory_guard, check_memory_leaks, compute-sanitizer wrapper + plugin.py # pytest hooks: markers, fixtures, terminal summary, pyproject loader + assertions/ # assert_close(), tolerances (ContextVar overrides + MPS overlay), Rich reports + backends/ # Backend Protocol + CUDABackend + MPSBackend (deadlock-safe events) [v1.0] + decorators/ # @dtypes, @shapes, @devices (CUDA+MPS), @parametrize_gpu (stride_categories=) + fixtures/ # gpu_benchmark (CUDA events / MPS device-sync), memory_tracker, gpu_device + fuzzing/ # fuzz_shapes(), fuzz_strides() [v1.0], ShapeStrategy, StrideStrategy + sanitizers/ # memory_guard, check_memory_leaks, compute-sanitizer wrapper, determinism [v1.0] arch/ # GPU detection (pynvml/torch), @require_arch, tensor cores analysis/ # roofline model, regression detection (Mann-Whitney U), bottleneck - reporting/ # Rich console, JSON, CI (JUnit XML, GitHub annotations, PR comments) + reporting/ # Rich console, JSON, HTML dashboard [v1.0], CI (JUnit XML, GH annotations, PR comments) ``` ## Build & Test @@ -42,32 +44,46 @@ mypy src/ # Type check - **Statistical benchmarking:** CUDA events + L2 flush + IQR outlier removal - **Shape fuzzing priority:** degenerate > non-tile-aligned > prime > power-of-2 boundary > large > mixed +## v1.0 highlights (delivered) + +Four parallel engineering tracks shipped in v1.0: + +- **Track A — MPS backend** (`feat/track-a-mps`): `gpucheck.backends.{Backend, CUDABackend, MPSBackend}`, `@devices("mps")`, deadlock-safe MPS event timing (avoids pytorch#162872), 2× MPS tolerance overlay (PROVISIONAL — pending M-machine P99 calibration), 12-entry `[tool.gpucheck.mps.xfail]` config block citing concrete pytorch issues, `[mps]` and `[apple]` install extras (pin `torch>=2.6`). +- **Track B — Stride fuzzing** (`feat/track-b-strides`): `fuzz_strides()` deterministic 7-category corpus (row-major / column-major / broadcast-induced / transpose / slice / contiguous-after-clone / gather-induced), `StrideStrategy` for Hypothesis, `parametrize_gpu(stride_categories=...)` wiring. +- **Track C — Thread-safety** (`feat/track-c-thread-safety`): `tolerance_context()` now backed by `contextvars.ContextVar` (safe for `pytest-xdist` workers, threads, asyncio). Also mitigates security finding TM-E1 by adding `realpath`+allowlist validation for `CUDA_HOME`/`CUDA_PATH` in `sanitizers/race.py`. +- **Track D — Release bundle** (`feat/track-d-bundle`): HTML dashboard reporter, `assert_deterministic` / `@requires_determinism` / `DeterminismError`, committed `uv.lock` (DEP-1 mitigation), CI permissions hardening (CFG-2 mitigation), reporting test coverage 0% → 98%. + +Test count: **224 passing** (was 117 pre-v1.0). + +See `CHANGELOG.md`, `MIGRATION.md`, `.claude/teams/engineering/v1.0/DIFF_LOG.md`, `.claude/teams/research/v1.0/SYNTHESIS.md`, and `.claude/teams/security/v1.0/FINDINGS.md`. + ## Strengths -- Found 8 real bugs in Triton/PyTorch with 511 test configs +- Found 8 bugs in Triton/PyTorch with 511 test configs (2 externally verified: triton#9838 open, triton#9839 closed; 6 internal-ledger findings reproducible from `examples/`) - 83% error in Triton layer norm (triton#9838), FP16 drift in tutorial matmul (triton#9839) - Clean pytest plugin architecture with proper hook registration - Comprehensive dtype coverage including FP8 (E4M3, E5M2) - Rich mismatch reports with error histograms -- Hypothesis integration via ShapeStrategy +- Hypothesis integration via ShapeStrategy and StrideStrategy - Architecture detection: Pascal through Blackwell (SM60-SM120) - Tensor core generation tracking with GTX 16xx exclusion +- **Apple MPS backend with curated xfail registry and deadlock-safe benchmarking** (v1.0) +- **Stride / contiguity fuzzing across 7 categories** (v1.0) +- **Thread-safe tolerance overrides via ContextVar** (v1.0) +- **HTML dashboard, determinism sanitizer, committed uv.lock** (v1.0) +- **Reporting module 98% covered** (v1.0; was 0%) +- **Published CHANGELOG, CONTRIBUTING, MIGRATION** (v1.0) ## Known Weaknesses & Gaps -- No stride/contiguity fuzzing (only shapes and values) -- No AMD ROCm or Intel XPU support -- No GPU CI (tests run CPU-only on GitHub Actions) +- No AMD ROCm or Intel XPU support (planned) +- No GPU CI (tests run CPU-only on GitHub Actions; GPU validation runs locally on author hardware) - No profiling integration (Nsight Compute/Systems) -- No determinism testing support - No gradient/backward pass testing - No multi-GPU communication testing (NCCL) - No CUDA graph testing support -- No HTML/dashboard reporting -- Reporting module (console, json, ci) has zero test coverage -- Thread-safety issue in tolerance override stack -- Memory leak detection uses process-level metrics (imprecise) -- No changelog, no contributing guide, no migration docs +- Memory leak detection uses process-level metrics on MPS (psutil RSS proxy per pytorch#164299) — imprecise but deliberate fallback +- The 2× MPS tolerance multiplier is PROVISIONAL — pending P99 calibration on Akash's M-machine; ops with measured drift > 2× will move to the xfail registry rather than further inflate the multiplier ## Code Standards @@ -81,11 +97,12 @@ mypy src/ # Type check ## Git Conventions -- Branch: feature/, fix/, refactor/, docs/ -- Commits: conventional commits (type(scope): description) -- Never commit to main directly +- Branch: `feat/`, `fix/`, `refactor/`, `docs/`, `test/`, `perf/`, `chore/` +- Commits: **Conventional Commits 1.0** going forward (`feat(scope): message`); legacy `[ Type ] :` bracket-style commits in pre-v1.0 history are unchanged +- Never commit to `main` directly - One logical change per commit - Account: Akasxh / drakathakash@gmail.com +- See `CONTRIBUTING.md` for the full PR + commit-message guide ## Expert System @@ -98,6 +115,7 @@ mypy src/ # Type check | fixtures/ | pytest-plugin-architect | performance-engineer | | fuzzing/ | fuzzing-property-testing-lead | numerical-analysis-specialist | | sanitizers/ | security-safety-specialist | cuda-systems-engineer | +| backends/ | cuda-systems-engineer | api-design-dx-lead | | arch/ | cuda-systems-engineer | triton-compiler-specialist | | analysis/ | performance-engineer | numerical-analysis-specialist | | reporting/ | docs-developer-advocate | cicd-release-engineer | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1a3bb45 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,341 @@ +# Contributing to gpucheck + +Thanks for your interest in **gpucheck**. This document describes how to set +up a development environment, the project's testing conventions, the PR +process, the commit-message format, and how to file effective bug reports. + +--- + +## Reporting bugs + +1. Check the [open issues](https://github.com/Akasxh/gpucheck/issues) first. + If a duplicate exists, add information rather than open a new one. +2. Include: + - **gpucheck version**: + `python -c "import gpucheck; print(gpucheck.__version__)"` + - **PyTorch version** and **device backend** — CUDA driver/toolkit + version for NVIDIA, macOS + chip family for Apple Silicon. + - A **minimal reproducer** — ideally a `pytest` test that uses + `gpucheck` and fails. The `examples/triton_layernorm_bug.py` and + `examples/triton_matmul_bug.py` files are the reference style. + - The full traceback or `assert_close` mismatch report. +3. If the bug is in an upstream kernel (Triton, PyTorch CUDA op, MPS op) + that gpucheck merely surfaced, file the upstream issue first and + cross-link. + +--- + +## Development setup + +gpucheck targets **Python ≥ 3.10**. The project uses +[`uv`](https://docs.astral.sh/uv/) for dependency management and ships a +committed `uv.lock` for reproducible installs. + +```bash +git clone https://github.com/Akasxh/gpucheck.git +cd gpucheck + +# Reproducible install from the lockfile (matches CI exactly) +uv sync --frozen + +# Or, an editable install with all dev tools (matches local dev): +uv pip install -e ".[dev]" +``` + +If you do not want to use `uv`, the equivalent `pip` flow still works: + +```bash +pip install -e ".[dev]" +``` + +Optional extras (see `pyproject.toml`): + +| Extra | Purpose | +|---|---| +| `gpucheck[torch]` | Full PyTorch-backed assertions, fixtures, fuzzing | +| `gpucheck[mps]` | Apple Silicon Metal Performance Shaders backend (`torch>=2.6`) | +| `gpucheck[apple]` | Alias for `[mps]` | +| `gpucheck[hypothesis]` | Property-based shape / tensor / stride strategies | +| `gpucheck[cupy]` | CuPy interop for `__cuda_array_interface__` paths | +| `gpucheck[triton]` | Helpers for Triton-kernel test files | +| `gpucheck[all]` | Equivalent to `[torch, hypothesis]` | +| `gpucheck[dev]` | All of `[all]` plus ruff, mypy, pytest-cov | + +To run the full check loop: + +```bash +ruff check src/ tests/ +mypy src/ +uv run pytest --tb=short -q +``` + +These three commands gate every PR. + +--- + +## Running tests + +```bash +# Unit suite (CPU-only — runs everywhere, including CI) +uv run pytest -q + +# GPU-required tests (CUDA host) +uv run pytest tests/gpu_integration/ -v + +# Examples (also collected as tests) +uv run pytest examples/ -v +``` + +- The unit suite runs on every CI box. CI is currently CPU-only on GitHub + Actions; GPU validation runs locally on author hardware. +- GPU integration tests are auto-skipped via `pytest.mark.gpu` and + `pytest.mark.multi_gpu` when no GPU is detected. +- **MPS tests run automatically on Apple Silicon hosts.** When + `torch.backends.mps.is_available()` returns `True`, `@devices()` + expands to include `"mps"`, and `tests/test_devices_mps.py` / + `tests/test_assert_close_mps.py` / `tests/test_mps_xfail.py` exercise + the MPS backend without further configuration. On non-Apple hosts the + same tests collect but skip cleanly. +- The full test count as of v1.0.0rc1 is **224 passing** (CPU-only). + +--- + +## Code style + +| Concern | Tool | Config | +|---|---|---| +| Formatter / linter | `ruff` | `pyproject.toml` `[tool.ruff]` | +| Type checking | `mypy --strict` | `pyproject.toml` `[tool.mypy]` | +| Line length | 100 chars | `[tool.ruff].line-length = 100` | +| Python target | `py310` | `[tool.ruff].target-version = "py310"` | +| Imports of optional deps | **lazy** — never at module top-level | See `src/gpucheck/__init__.py` for the canonical pattern | + +**Lazy-import rule (load-bearing):** `torch`, `pynvml`, `hypothesis`, and +`cupy` must never be imported at collection time. New code must follow the +existing pattern: resolve at test execution, not at decoration; or use +the lazy module-level `__getattr__` shim in `src/gpucheck/__init__.py`. + +**Error handling:** no bare `except:`; specific exception classes only +(`ImportError`, `RuntimeError`, `pynvml.NVMLError`, etc.). + +--- + +## Commit message format + +**Going forward, gpucheck uses [Conventional Commits 1.0](https://www.conventionalcommits.org/).** +The four track commits that landed v1.0 already follow this: + +``` +feat(mps): add Apple Silicon MPS backend with deadlock-safe benchmarking +feat(fuzzing): stride and contiguity fuzzing for GPU kernels +fix(tolerances): thread-safe override stack via contextvars; mitigate TM-E1 +feat(reporting+sanitizers): HTML dashboard, determinism, lockfile, CI hardening +chore(lock): regenerate uv.lock post-merge to include Track-A [mps] extra +``` + +**Legacy `[ Type ] :` bracket-style commits** (visible in pre-v1.0 +history, e.g. `[ Fix ] : resolve 7 bugs`) are unchanged — we are not +rewriting history. New commits should use Conventional Commits. + +Allowed types: `feat`, `fix`, `docs`, `test`, `perf`, `refactor`, `chore`, +`build`, `ci`, `style`. Scope is optional but encouraged for cross-cutting +changes (e.g. `feat(mps)`, `fix(tolerances)`). + +Rules: + +- One logical change per commit. +- The first line is < 72 chars. +- Body (optional) wraps at 72 chars and explains *why*, not *what*. +- Reference issues with `Closes #N`, `Fixes #N`, or `triton#NNNN` / + `pytorch#NNNN` for upstream cross-references. +- **Never commit directly to `main`.** Use a feature branch. + +--- + +## Branch naming + +``` +feat/ # new feature +fix/ # bug fix +refactor/ # internal cleanup, no behaviour change +docs/ # docs-only +test/ # test-only +perf/ # performance work +chore/ # tooling, deps, CI +``` + +For the v1.0 release, four parallel tracks lived on +`feat/track-{a,b,c,d}-…` (MPS / strides / thread-safety / bundle). + +--- + +## Pull request process + +1. Open a draft PR early — describe what you intend to do before doing it. + Cite the file:line you plan to touch. +2. Keep the PR scoped. If it grows past ~400 changed lines, split. +3. Each PR must pass: + - `ruff check src/ tests/` + - `mypy src/` + - `uv run pytest --tb=short -q` + - `uv run pytest examples/ -v` *(if examples are touched)* + On a GPU host, also run `uv run pytest tests/gpu_integration/`. +4. Add a CHANGELOG entry under `## [Unreleased]` in the appropriate + subsection (Added / Changed / Fixed / etc.). Each release rolls + `[Unreleased]` into the next release section. +5. Update `MIGRATION.md` if the change is breaking. +6. The PR description must include: + - **Why** the change is needed. + - **What** behaviour changes (with `before` / `after` if numeric). + - **Test plan** — explicit list of commands run and on what hardware. + +Reviewers look for: + +- Lazy-import discipline preserved. +- Public symbols documented. +- No regression in mypy strict pass. +- Tolerance changes traceable to a measured benchmark, not folk wisdom. +- For MPS-affecting changes: no `Event.synchronize()` call sites + (deadlock — pytorch#162872); `torch.mps.synchronize()` only. + +--- + +## Architecture overview + +``` +src/gpucheck/ +├── __init__.py # Lazy public API — zero torch/pynvml at import +├── plugin.py # pytest hooks: addoption, configure, collection_modify, terminal_summary +├── assertions/ # assert_close + dtype-aware tolerances + Rich mismatch reports +├── backends/ # Backend Protocol; CUDA + MPS implementations (v1.0) +├── decorators/ # @dtypes / @shapes / @devices / @parametrize_gpu +├── fixtures/ # gpu_benchmark / gpu_device / memory_tracker +├── fuzzing/ # fuzz_shapes / fuzz_strides / Hypothesis strategies +├── sanitizers/ # memory_guard / determinism / compute-sanitizer wrapper +├── arch/ # detect_gpus / @require_arch / @require_capability / tensor cores +├── analysis/ # roofline / regression / bottleneck classification +└── reporting/ # ConsoleReporter / JSONReporter / HTMLReporter / CI annotations +``` + +Detailed design notes live in `CLAUDE.md` § "Key Design Decisions". + +--- + +## How to add a new public API + +1. Land the implementation behind a feature flag or extra if it adds a + new optional dependency. +2. Export from the relevant submodule's `__all__` and add a lazy entry + to `src/gpucheck/__init__.py`'s `_LAZY_MAP` if it should be importable + as `gpucheck.foo`. +3. Add a docstring (Numpy or Google style — match neighbouring code in + the same file). +4. Add unit tests under `tests/` and, if GPU-only, under + `tests/gpu_integration/`. +5. Add a CHANGELOG `Added` entry under `[Unreleased]`. + +--- + +## How to add a new GPU backend + +The v1.0 MPS backend is the reference. To add a new backend (e.g. ROCm, +XPU): + +1. Implement the `Backend` Protocol from + `src/gpucheck/backends/_protocol.py`. Methods to provide include + device enumeration, allocator stats, and an `event_timer` context + manager. +2. Register in `src/gpucheck/backends/__init__.py` so + `available_backends()` and `get_backend(name)` discover the new + implementation. +3. Extend `_is_device_available` and `_detect_devices` in + `src/gpucheck/decorators/devices.py` to recognize the new device-type + string. +4. Widen the `assert_close` GPU fast-path (`assertions/close.py`) and the + `gpu_benchmark` runner (`fixtures/benchmark.py`) to dispatch on the + new device type. +5. If the backend has known broken kernels, add a + `[tool.gpucheck..xfail]` block in `pyproject.toml` and a + loader in `src/gpucheck/plugin.py`. +6. Mirror the test pattern: `tests/test_backends.py`, + `tests/test_devices_.py`, `tests/test_assert_close_.py`, + `tests/test__xfail.py`. + +--- + +## Releasing + +The release flow uses TestPyPI as a staging gate: + +1. Bump `version` in `pyproject.toml` and `__version__` in + `src/gpucheck/__init__.py`. Keep them in lock-step. +2. Update `CHANGELOG.md`: roll `[Unreleased]` into the new version + section with a date. +3. Update `MIGRATION.md` if the release introduces user-visible API + shape changes. +4. Tag the release on `release/v` (e.g. `git tag v1.0.0rc1`). +5. Build and upload to TestPyPI first; verify install + smoke test; + then promote to PyPI. (Note: `~/.pypirc` is currently a manual + author-only step.) +6. Create a GitHub Release citing the CHANGELOG entry; attach signed + wheel + sdist checksums. + +--- + +## Security + +Security issues should NOT be reported through public GitHub issues. +Instead, email **** with subject +`gpucheck security: `. + +The current security audit ledger lives in +`.claude/teams/security/v1.0/FINDINGS.md` (0 CRITICAL · 0 HIGH · +3 MEDIUM mitigated · 13 LOW advisory). + +--- + +## Expert system + +`.claude/experts/` holds 10 domain-specific personas (numerical-analysis, +pytest-plugin, fuzzing, security, CUDA-systems, performance, docs, +CI/CD, API-design, Triton). When working on a module, consult the +relevant expert(s): + +| Module | Primary | Secondary | +|---|---|---| +| `assertions/` | numerical-analysis | api-design-dx-lead | +| `decorators/` | pytest-plugin-architect | api-design-dx-lead | +| `fixtures/` | pytest-plugin-architect | performance-engineer | +| `fuzzing/` | fuzzing-property-testing-lead | numerical-analysis | +| `sanitizers/` | security-safety-specialist | cuda-systems-engineer | +| `backends/` | cuda-systems-engineer | api-design-dx-lead | +| `arch/` | cuda-systems-engineer | triton-compiler-specialist | +| `analysis/` | performance-engineer | numerical-analysis | +| `reporting/` | docs-developer-advocate | cicd-release-engineer | +| `plugin.py` | pytest-plugin-architect | api-design-dx-lead | +| CI/CD | cicd-release-engineer | security-safety-specialist | +| `examples/` | docs-developer-advocate | triton-compiler-specialist | +| `pyproject.toml` | cicd-release-engineer | api-design-dx-lead | + +--- + +## Engineering Team protocol (claude-forge contributors) + +If you contribute via [claude-forge](https://claude.com/claude-code) and +want to dispatch the Engineering Team for a substantial change, the +binding protocol lives at `~/.claude/teams/engineering/PROTOCOL.md`. +The four v1.0 tracks were produced under this protocol; the audit trail +is in `.claude/teams/engineering/v1.0/`. + +The Documentation Team protocol is at `~/.claude/teams/docs/PROTOCOL.md`, +with v1.0 docs evidence at `.claude/teams/docs/v1.0/`. + +--- + +## Project meta + +- **Author / maintainer:** Akash +- **License:** Apache-2.0 +- **Issues:** +- **PyPI:** +- **Source:** diff --git a/DIFF_LOG.md b/DIFF_LOG.md new file mode 100644 index 0000000..0c221af --- /dev/null +++ b/DIFF_LOG.md @@ -0,0 +1,28 @@ +# DIFF_LOG.md — Phase A executor (close.py collision group) + +Owner: engineering-executor +Branch: release/v1.0 +Collision group: T-01 → T-02 → T-10 (serialized per skeptic C2) + +## Iteration 1 — Task T-01: Top-level torch import defeats lazy-import contract +- **File**: `src/gpucheck/assertions/close.py` +- **Change**: Replaced module-level `try: import torch as _torch / _has_torch` block with a lazy `_torch_mod()` cached helper; rewrote `device_type` detection and the GPU fast-path to bind `_torch = _torch_mod()` at the top of `assert_close`, then guard with `_torch is not None`. +- **Reason**: CLAUDE.md Lazy-imports decision-record requires torch to never be imported at collection time. The previous `try: import torch as _torch` ran at module load, breaking the contract. Caching via a sentinel makes the lookup happen at most once per process. +- **Acceptance criterion addressed**: IMPLEMENTATION_PLAN_v1.1 T-01 — `python -c "import gpucheck.assertions.close; import sys; assert 'torch' not in sys.modules"` now passes; existing `tests/test_assertions.py` still green (32 passed). + +## Iteration 2 — Task T-02: Add `.contiguous()` to `_to_numpy` slow path +- **File**: `src/gpucheck/assertions/close.py` +- **Change**: Inserted `.contiguous()` between `.cpu()` and `.numpy()` on both torch.Tensor branches of `_to_numpy` (the primary `hasattr(tensor, "detach")` branch and the dlpack fallback inside the `__cuda_array_interface__` block). Added a comment citing PM-4. +- **Reason**: torch <2.1 raises `RuntimeError: input array is not C-contiguous` when `.numpy()` is called on stride-fuzzed / sliced / transposed tensors. Preventive even on newer torch — known to fire on older PyTorch. +- **Acceptance criterion addressed**: security-postmerge PM-4 / planner T-02 — new `tests/test_assert_close_contiguous.py` exercises 3 stride patterns (slice, transpose, broadcast) × 2 entry points (`_to_numpy` directly and `assert_close` end-to-end); all 6 cases pass. + +- **File**: `tests/test_assert_close_contiguous.py` (created) +- **Change**: New parametrized test module with 6 cases pinning the contiguity fix. +- **Reason**: T-02 acceptance demands a regression test that would fail without the `.contiguous()` insertion. +- **Acceptance criterion addressed**: planner T-02. + +## Iteration 3 — Task T-10: Pin numeric fields in mismatch report +- **File**: `tests/test_assertions.py` +- **Change**: Added `TestMismatchReportPinnedNumerics` class with 3 tests pinning (a) max abs error / mean abs error / row labels (b) mismatch count "5 / 6 (83.33%)" + 2-D max-error location "(1, 2)" (c) histogram bucket label "[1e-3, 1e-2)" with count 3 after stripping ANSI escapes. +- **Reason**: mutator-survivors top-leverage #1 — `assertions/reporting.py` had ~30 surviving mutants because no test asserted exact numeric values from the report. Hard-coded values + 2-D index + ANSI-aware bar count force any arithmetic substitution / unravel-axis swap / bucket-formatter mutation to fail. +- **Acceptance criterion addressed**: planner T-10 / mutator-survivors top-leverage #1 — adds 3 tests (commit-message claim) without modifying `reporting.py` source. diff --git a/EXECUTOR_LOG.md b/EXECUTOR_LOG.md new file mode 100644 index 0000000..afd56ef --- /dev/null +++ b/EXECUTOR_LOG.md @@ -0,0 +1,32 @@ +# EXECUTOR_LOG.md + +Branch: release/v1.0 +Owner: engineering-executor (Phase A close.py collision group) + +## T-01 — Top-level torch import defeats lazy-import contract +- Test count: 224 passed, 1 skipped (pre-T-01) → 224 passed, 1 skipped (post-T-01); no behavioural change to assertion-suite. +- Lazy-import contract: `python -c "import gpucheck.assertions.close; import sys; assert 'torch' not in sys.modules"` PASS. +- Mutmut killed (baseline): 169. +- Commit SHA: 9f430f7. + +## T-02 — `.contiguous()` on slow path for stride-fuzzed tensors (PM-4) +- Test count: 224 passed, 1 skipped → 230 passed, 1 skipped (+6 new parametrized cases). +- Lazy-import contract: re-verified PASS (no torch in sys.modules after `import gpucheck.assertions.close`). +- New file: `tests/test_assert_close_contiguous.py`. +- Commit SHA: 1dd7ba9. + +## T-10 — Pin numeric fields in mismatch report (kills ~30 mutants) +- Test count: 230 passed, 1 skipped → 233 passed, 1 skipped (+3 new pinned-numerics tests). +- Lazy-import contract: re-verified PASS. +- Lint: `uv run ruff check src/ tests/` clean. +- Type check: `uv run mypy src/` clean (Success: no issues found in 41 source files). +- Mutmut killed (post-T-10, no full re-run): 169 (unchanged baseline; full re-run intentionally skipped per task instructions). The new tests target reporting.py mutants that are expected to flip ~30 from `survived` → `ok_killed` on the next mutmut sweep. +- Commit SHA: b500339. + +## Final summary +- Commits (in order): `9f430f7` (T-01), `1dd7ba9` (T-02), `b500339` (T-10). +- Final test count: **233 passed, 1 skipped** (was 224 passed, 1 skipped at branch tip — net +9 tests across T-02 (+6) and T-10 (+3)). +- Mutmut kill count: 169 (unchanged; full re-run not requested per task spec — new tests will be measured on the next sweep). +- Lazy-import contract: PASS. +- ruff: clean. mypy: clean. +- No push performed (orchestrator owns push). diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 0000000..4db794f --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,404 @@ +# Migrating from gpucheck v0.1.0 → v1.0 + +This guide covers users on `gpucheck >= 0.1, < 1.0` upgrading to +`gpucheck >= 1.0`. + +**TL;DR — v1.0 is API-additive.** Existing CUDA-only test suites continue +to work without modification. The new entry points are the +`gpucheck.backends` package, `@devices("mps")` / `@devices("all")`, the +`[tool.gpucheck.mps.xfail]` config block, and the `[mps]` install extra. +The 2× MPS tolerance multiplier is **PROVISIONAL** pending M-machine +calibration. + +The version jump from `0.1.0` straight to `1.0.0rc1` is intentional: +v0 was an alpha PyPI publish (`Development Status :: 3 - Alpha`); v1 is +the first release-quality tag with a published CHANGELOG, MIGRATION, +CONTRIBUTING guide, and committed lockfile. + +--- + +## Quick checklist + +| If you currently … | You will need to … | +|---|---| +| Pin `gpucheck<1` | Read this file before bumping | +| Use `@devices("cuda:0")` only | No change required | +| Want MPS coverage | Add `pip install gpucheck[mps]` and use `@devices()` (auto) or `@devices("mps")` | +| Use `@devices("all")` | Behaviour expanded — now includes MPS on Apple Silicon | +| Construct a `GPUInfo` directly in tests | Note new `backend: str = "cuda"` field — see §1 | +| Call `gpucheck.arch.detection.detect_gpus()` directly | Still works; the v1 API surface is `gpucheck.backends.get_backend()` — see §2 | +| Wrap test code in `tolerance_context(...)` from `pytest-xdist` workers or threads | Now safe — see §4 | +| Have an op known-broken on MPS | Add it to `[tool.gpucheck.mps.xfail]` in `pyproject.toml` — see §5 | +| Want stride / contiguity coverage | Use `fuzz_strides()` or `parametrize_gpu(stride_categories=...)` — see §7 | +| Want reproducible installs | `uv sync --frozen` against the committed `uv.lock` — see §9 | + +--- + +## 1. New `Backend` Protocol + +v1.0 introduces a `runtime_checkable` `Backend` Protocol at +`src/gpucheck/backends/_protocol.py`, with two implementations: +`CUDABackend` (`backends/cuda.py`) and `MPSBackend` (`backends/mps.py`). + +The Protocol is the **v1 public API for backend introspection**: + +```python +from gpucheck import available_backends, get_backend + +# v1.0 — list backends present on this host +print(available_backends()) # e.g. ("cuda",) or ("cuda", "mps") or ("mps",) + +# Get a specific backend +cuda = get_backend("cuda") +mps = get_backend("mps") # raises if MPS not available + +# Use the backend's deadlock-safe event timer +with mps.event_timer() as timer: + run_kernel() +print(timer.elapsed_ms()) +``` + +### Compatibility note + +If you wrote against `gpucheck.arch.detection.detect_gpus()` directly, +**that still works** — the v1 `Backend` Protocol is a higher-level entry +point, not a replacement. `detect_gpus()` continues to return a list of +`GPUInfo`. New code should prefer `gpucheck.backends.get_backend()`. + +--- + +## 2. `GPUInfo` shape extension + +The `GPUInfo` dataclass in `src/gpucheck/arch/detection.py` gains a single +new field: + +```python +@dataclass(frozen=True, slots=True) +class GPUInfo: + device_id: int + name: str + compute_capability: tuple[int, int] + architecture: str + memory_total_mb: int + memory_free_mb: int + driver_version: str + cuda_version: str + supports_fp16: bool + supports_bf16: bool + supports_fp8: bool + supports_tf32: bool + tensor_core_generation: int | None + max_shared_memory_per_block: int + backend: str = "cuda" # NEW in v1.0 — defaults to "cuda" for back-compat +``` + +**Migration:** + +- All existing fields are unchanged. CUDA-only code is unaffected. +- `backend` defaults to `"cuda"` — code that constructs `GPUInfo(...)` + without keyword arguments and relies on positional ordering of the + trailing field is **not** broken because the new field is at the + end with a default. +- On MPS hosts, `compute_capability`, `cuda_version`, and + `tensor_core_generation` carry backend-appropriate sentinel values + (e.g. `(0, 0)`, `""`, `None`). Code that branches on + `gpu.compute_capability >= (8, 0)` should now also check + `gpu.backend == "cuda"`: + +```python +# v0.1.0 +gpu = detect_gpu() +if gpu and gpu.compute_capability >= (8, 0): + enable_bf16_path() + +# v1.0 — same code still runs unchanged on CUDA hosts. +# To handle MPS too: +gpu = detect_gpu() +if gpu and gpu.backend == "cuda" and gpu.compute_capability >= (8, 0): + enable_bf16_path_cuda() +elif gpu and gpu.backend == "mps": + enable_mps_path() +``` + +--- + +## 3. `@devices("mps")` and `@devices("all")` + +In v0, `@devices()` with no arguments auto-detected CUDA devices only. +In v1, **`@devices()` additionally includes `"mps"`** when +`torch.backends.mps.is_available()` returns `True`, and `@devices("all")` +resolves to the union of all detected devices. + +```python +# v0 behaviour (still supported) +@devices("cuda:0") +def test_kernel(device): ... + +# v1 — auto-detect picks up CUDA AND MPS +@devices() +def test_kernel(device): ... +# device parametrizes over ["cuda:0", "mps"] on a hybrid host + +# Explicit MPS +@devices("mps") +def test_metal_kernel(device): ... + +# Mix +@devices("cuda:0", "mps") +def test_both(device): ... + +# All — explicit +@devices("all") +def test_everywhere(device): ... +``` + +Tests for unavailable devices skip cleanly via `_is_device_available`. + +--- + +## 4. `tolerance_context()` is now thread-safe + +In v0, `src/gpucheck/assertions/tolerances.py` used a module-level list +as the override stack with the inline comment `# NOT thread-safe`. In +v1, the override stack is a `contextvars.ContextVar`: + +- `pytest-xdist` workers (already process-isolated) continue to work. +- Concurrent threads inside a single worker no longer leak overrides + across threads. +- `asyncio` tasks are isolated per the standard `ContextVar` semantics. + +**No user-facing API change** — the same `tolerance_context(atol, rtol)` +context manager works exactly as before. Prior code is unchanged. + +```python +from gpucheck import tolerance_context + +with tolerance_context(atol=1e-3, rtol=1e-3): + assert_close(actual, expected) # ← uses overridden tolerances +``` + +--- + +## 5. Per-kernel xfail registry + +v1 introduces a curated registry of known-broken MPS kernels in +`pyproject.toml`: + +```toml +[tool.gpucheck.mps.xfail] +ops = [ + "scaled_dot_product_attention.large", # pytorch#179352 + "scaled_dot_product_attention.backward", # pytorch#179294 + "layer_norm.backward.shape1", # pytorch#173525 + "batch_norm.backward.channels_last", # pytorch#175189 + "conv2d.large_channels", # pytorch#142836 + "conv2d.backward.channels_last_format", # pytorch#174269 + "F.linear.backward.bf16_3d_nobias_m5", # pytorch#181936 + "softmax.large_attention", # pytorch#96602 + "avg_pool2d.backward.channels_last", # pytorch#175190 + "binary_ops.uint16_uint32_uint64", # pytorch#176296 + "BCE_loss", # pytorch#137001 + "matmul.backward.over_32K_elements", # pytorch#177116 +] +``` + +These 12 entries are drawn from the research SYNTHESIS top-impact open +MPS bugs (see `.claude/teams/research/v1.0/SYNTHESIS.md` §Sub-Q 2). + +Public API: + +```python +import gpucheck +gpucheck.is_mps_xfailed("softmax.large_attention") # → True +gpucheck.mps_xfail_list() # → ("scaled_dot_product_attention.large", ...) +gpucheck.register_mps_xfail("my_custom.broken_op") # programmatic registration +``` + +The xfail registry is a **living document**. Add entries as you discover +new broken kernels; remove entries as upstream issues close. Tolerance +multipliers cannot rescue silent-correctness or crash bugs — that is +what the registry is for. + +--- + +## 6. PROVISIONAL 2× MPS tolerance multiplier + +The default tolerances in `src/gpucheck/assertions/tolerances.py` are +calibrated against cuBLAS on Turing/Ampere. v1.0 ships with an MPS +overlay that applies a **2× per-dtype multiplier** for FP32, FP16, and +BF16 on top of the CUDA baseline: + +| dtype | CUDA atol | CUDA rtol | MPS atol | MPS rtol | Multiplier | +|---|---|---|---|---|---| +| float32 | 1e-4 | 1e-4 | 2e-4 | 2e-4 | 2× | +| float16 | 1e-2 | 1e-2 | 2e-2 | 2e-2 | 2× | +| bfloat16 | 5e-2 | 5e-2 | 1e-1 | 1e-1 | 2× | +| float64 | 1e-10 | 1e-7 | unchanged | unchanged | 1× | + +**This 2× multiplier is PROVISIONAL.** It is grounded in two arguments — +the FlashAttention precision-floor precedent and the absence of FP16 +tensor cores on Apple Silicon — but it has not yet been calibrated +against P99 measured drift on Akash's M-machine. See +`.claude/teams/research/v1.0/SYNTHESIS.md` §Sub-Q 7 for the full +calibration plan. + +If your kernel exhibits drift > 2× on MPS, the current guidance is to +**move the op to the xfail registry rather than further inflate the +multiplier**. The multiplier covers precision-floor noise; the registry +covers implementation bugs. + +The number may change in v1.0.0 final after calibration. Users hard-coding +overlays on top of v1.0.0rc1 should track this CHANGELOG section. + +**v0 → v1 migration:** none required for code that uses +`assert_close` defaults. If you previously hard-coded MPS-friendly +multipliers, you can drop them. To force CUDA-style tolerances on an +MPS tensor, use `tolerance_context(atol=..., rtol=...)`. + +--- + +## 7. Stride / contiguity fuzzing — new public API + +v1 adds a 7-category stride corpus at `src/gpucheck/fuzzing/strides.py`: + +```python +from gpucheck.fuzzing import fuzz_strides, fuzz_strides_for_category, STRIDE_CATEGORIES + +# Deterministic corpus across all categories +strides = fuzz_strides(shape=(8, 16, 32), dtype=torch.float32, seed=42) + +# One specific category (snake_case is canonical; kebab-case aliases +# such as "broadcast-induced" are accepted with a DeprecationWarning). +broadcast = fuzz_strides_for_category( + shape=(8, 16, 32), dtype=torch.float32, category="broadcast", +) + +# Hypothesis property-based testing +from gpucheck.fuzzing import StrideStrategy +from hypothesis import given + +@given(stride=StrideStrategy(shape=(8, 16, 32))) +def test_kernel_any_stride(stride): ... + +# Wired into parametrize_gpu +from gpucheck import parametrize_gpu + +@parametrize_gpu( + dtypes=("float32",), + shapes=((8, 16, 32),), + devices=("cuda:0",), + stride_categories=("row_major", "transpose", "broadcast"), +) +def test_my_kernel(dtype, shape, device, stride_category, stride): ... +``` + +The 7 categories (canonical snake_case): `row_major`, `column_major`, +`broadcast`, `transpose`, `slice`, `non_contig`, `gather`. The previous +kebab-case spellings (`row-major`, `broadcast-induced`, +`contiguous-after-clone`, `gather-induced`) are still accepted but emit +a `DeprecationWarning` and route to the snake_case form. + +Pre-v1.0, the project documented this gap as "No stride/contiguity +fuzzing (only shapes and values)" in `CLAUDE.md` "Known Weaknesses". +That gap is closed. + +--- + +## 8. Determinism sanitizer + +v1 adds `gpucheck.sanitizers.determinism`: + +```python +from gpucheck.sanitizers import assert_deterministic, requires_determinism, DeterminismError + +# Function form — fix seeds, re-run `n` times, compare outputs. +# `*args` / `**kwargs` are forwarded to my_kernel. +# Default mode is byte-identical (torch.equal); pass atol=/rtol= to +# opt into tolerance-based determinism (the right contract for MPS). +assert_deterministic(my_kernel, x, y, n=2) +assert_deterministic(my_kernel, x, y, n=3, atol=1e-5) # MPS-friendly + +# Decorator form — gate a test on determinism +@requires_determinism(n=3, seed=0) # byte-identical mode +def test_kernel_is_reproducible(): ... + +@requires_determinism(n=3, atol=1e-5) # tolerance mode (MPS) +def test_mps_kernel_is_quasi_deterministic(): ... +``` + +PyTorch's MPS docs are silent on determinism, and the empirical record +shows real run-to-run divergence (pytorch#181936, pytorch#170837, +pytorch#177116). gpucheck-MPS does **not** export a "deterministic +parity" guarantee. Use `assert_deterministic` to verify locally; do not +assume bit-exactness across runs. + +--- + +## 9. `uv.lock` committed (DEP-1 mitigation) + +v1.0 ships a committed `uv.lock`. CI installs from the lockfile via +`uv sync --frozen`. Downstream consumers benefit from supply-chain +reproducibility: + +```bash +# Reproducible install +uv sync --frozen + +# Equivalently, pin to the same dep set with pip: +uv export --no-dev > requirements.lock.txt +pip install -r requirements.lock.txt +``` + +The `pyproject.toml` floors continue to declare lower bounds; the +lockfile pins exact versions for the release. + +--- + +## 10. New install extras + +```bash +pip install gpucheck[mps] # MPS backend; pins torch>=2.6 +pip install gpucheck[apple] # alias for [mps] +pip install gpucheck[hypothesis] # property-based shape, tensor, stride strategies (no change) +``` + +The `[mps]` extra pins `torch>=2.6`, the floor at which +`torch.mps.synchronize()` is stable enough for the deadlock-safe +benchmark path (see CHANGELOG §Track A). + +--- + +## 11. `__version__` bump + +```python +# v0.1.0: src/gpucheck/__init__.py +__version__ = "0.1.0" + +# v1.0.0rc1: src/gpucheck/__init__.py +__version__ = "1.0.0rc1" +``` + +`pyproject.toml` `version` mirrors this. + +--- + +## 12. Removed / deprecated APIs + +**v1.0 removes nothing from v0.1.0.** It is API-additive in every case. + +If a future release deprecates a symbol, it will appear in this section +with a removal target version. + +--- + +## Cross-references + +- [`CHANGELOG.md`](./CHANGELOG.md) — full per-version diff with citations. +- [`CONTRIBUTING.md`](./CONTRIBUTING.md) — dev setup, PR process, + conventional-commits convention. +- `.claude/teams/research/v1.0/SYNTHESIS.md` — research substrate for + §1, §5, §6 (xfail registry, tolerance multipliers). +- `.claude/teams/engineering/v1.0/DIFF_LOG.md` — engineering substrate; + one row per file change across all four v1.0 tracks. +- `.claude/teams/security/v1.0/FINDINGS.md` — security ledger + (3 MEDIUM mitigated in v1.0: CFG-2, TM-E1, DEP-1). diff --git a/README.md b/README.md index abc1886..4b89523 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,11 @@ [![License](https://img.shields.io/github/license/Akasxh/gpucheck)](https://github.com/Akasxh/gpucheck/blob/main/LICENSE) [![CI](https://github.com/Akasxh/gpucheck/actions/workflows/ci.yml/badge.svg)](https://github.com/Akasxh/gpucheck/actions/workflows/ci.yml) -GPU kernel testing is painful. You write a CUDA kernel, eyeball `torch.allclose` with magic tolerances, and pray it works on a different GPU architecture. gpucheck is a pytest plugin that gives you dtype-aware assertions, parametric testing across dtypes/shapes/devices, CUDA-event benchmarking, shape fuzzing, and memory leak detection -- all from decorators and fixtures you already know how to use. +GPU kernel testing is painful. You write a CUDA or Metal kernel, eyeball `torch.allclose` with magic tolerances, and pray it works on a different GPU architecture. gpucheck is a pytest plugin that gives you dtype-aware assertions, parametric testing across dtypes/shapes/devices, GPU-event benchmarking, shape and stride fuzzing, and memory leak detection -- all from decorators and fixtures you already know how to use. **CUDA + Apple MPS** are first-class. -We tested gpucheck against Triton tutorials and PyTorch CUDA ops with **511 test configurations** and found **8 real bugs**, including a **83% error in Triton's layer norm** for non-power-of-2 dimensions ([triton#9838](https://github.com/triton-lang/triton/issues/9838)) and **FP16 accumulation drift in the tutorial matmul** ([triton#9839](https://github.com/triton-lang/triton/issues/9839)). +We tested gpucheck against Triton tutorials and PyTorch CUDA ops with **511 test configurations** and surfaced **8 bugs**, **2 of which are externally filed and verified** upstream: a **83% error in Triton's layer norm** for non-power-of-2 dimensions ([triton#9838](https://github.com/triton-lang/triton/issues/9838), open) and **FP16 accumulation drift in the tutorial matmul** ([triton#9839](https://github.com/triton-lang/triton/issues/9839), closed). The remaining 6 are internal-ledger findings reproducible from `examples/`.[1](#fn-bug-count) + +> **What's new in v1.0** — Apple Silicon MPS backend, stride/contiguity fuzzing, thread-safe tolerance overrides, HTML dashboard, determinism sanitizer, committed `uv.lock`. See [`CHANGELOG.md`](./CHANGELOG.md) for the full release notes and [`MIGRATION.md`](./MIGRATION.md) for the v0 → v1 upgrade guide. ```python import torch @@ -38,7 +40,8 @@ Optional dependencies for specific backends: ```bash pip install gpucheck[torch] # PyTorch + CUDA -pip install gpucheck[hypothesis] # Property-based shape fuzzing +pip install gpucheck[mps] # Apple Silicon (Metal Performance Shaders); pins torch>=2.6 +pip install gpucheck[hypothesis] # Property-based shape, tensor, and stride fuzzing pip install gpucheck[all] # Everything ``` @@ -335,6 +338,8 @@ gpucheck's shape fuzzing and dtype-aware testing found these real bugs in widely Most of these bugs were caught by non-power-of-2 shapes -- dimensions like 17, 127, 255 that hit tile boundary edge cases. This is exactly what `fuzz_shapes()` generates. +The table lists 5 of the 8 surfaced bugs in detail. The first two (`triton#9838`, `triton#9839`) are filed upstream and externally verified. The remaining 3 in the "8 bugs surfaced" total are internal-ledger findings not yet detailed in this table; reproducers live alongside the two examples below. + See [`examples/triton_layernorm_bug.py`](examples/triton_layernorm_bug.py) and [`examples/triton_matmul_bug.py`](examples/triton_matmul_bug.py) for standalone reproducers. ## Tested hardware and software @@ -364,7 +369,8 @@ gpucheck has been validated on the following hardware and software stack: **What is not yet tested on physical hardware:** - Ampere (A100, RTX 30xx), Ada (L40, RTX 40xx), Hopper (H100), and Blackwell GPUs are supported in the architecture detection and gating code but have only been tested via mocked GPU info, not on actual hardware. The tolerance model for these architectures is calibrated against published CUTLASS and cuBLAS error models. -- AMD ROCm and Intel XPU are not supported yet. The architecture detection module is NVIDIA-only for now. ROCm support is planned and would involve adding HIP detection via `torch.version.hip` and AMD GPU enumeration via `amdsmi` or `rocm_smi`. Intel XPU support would use `torch.xpu`. +- Apple Silicon is **supported via MPS as of v1.0** (see `gpucheck.backends.MPSBackend`, `@devices("mps")`, and the `[tool.gpucheck.mps.xfail]` config block). The 2× MPS tolerance multiplier is PROVISIONAL pending P99 calibration on M-machine — see [`CHANGELOG.md`](./CHANGELOG.md) and [`MIGRATION.md`](./MIGRATION.md) §6. +- AMD ROCm and Intel XPU are not supported yet. ROCm support would involve adding HIP detection via `torch.version.hip` and AMD GPU enumeration via `amdsmi` or `rocm_smi`. Intel XPU support would use `torch.xpu`. - Google TPU is not in scope for this project since TPUs use a fundamentally different programming model (XLA) that does not map to the kernel-level testing gpucheck provides. ## Comparison @@ -458,3 +464,8 @@ pytest examples/benchmark_example.py -v ## License Apache-2.0 + +--- + + +**[1]** "8 bugs / 2 externally verified" reconciles the prior README lead-claim with the visible "Bugs found" table. Source: `.claude/teams/research/v1.0/SYNTHESIS.md` §Sub-Q 8 (research team's archaeologist + empiricist evidence). The 2 externally verified bugs are `triton#9838` (open) and `triton#9839` (closed), both confirmed by upstream issue tracker as of 2026-05-01. diff --git a/pyproject.toml b/pyproject.toml index 6958b94..ec404a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "hatchling.build" [project] name = "gpucheck" -version = "0.1.0" -description = "pytest for GPU kernels — correctness, benchmarking, and fuzzing for CUDA and Triton" +version = "1.0.0rc1" +description = "pytest for GPU kernels — correctness, benchmarking, and fuzzing for CUDA, MPS, and Triton" readme = "README.md" license = "Apache-2.0" requires-python = ">=3.10" @@ -31,10 +31,21 @@ dependencies = [ "pytest>=7.0", "rich>=13.0", "numpy>=1.24", + # Python 3.10 lacks `tomllib` in the stdlib; the plugin's + # `_load_pyproject_config` falls back to `tomli`. Without this declared + # dependency, a fresh `pip install gpucheck` on 3.10 silently no-ops the + # `[tool.gpucheck.tolerances]` and `[tool.gpucheck.mps.xfail]` overlays + # (review BLOCKER S1; configuration-trust failure). + 'tomli>=2.0; python_version < "3.11"', ] [project.optional-dependencies] torch = ["torch>=2.0"] +# Apple Silicon MPS backend. torch>=2.6 is the floor where +# torch.mps.synchronize() is stable enough for gpucheck's deadlock-safe +# benchmark path (research SYNTHESIS §3, pytorch#162872 context). +mps = ["torch>=2.6"] +apple = ["gpucheck[mps]"] cupy = ["cupy-cuda12x>=13.0"] triton = ["triton>=3.0"] hypothesis = ["hypothesis>=6.0", "hypothesis[numpy]"] @@ -81,5 +92,50 @@ warn_return_any = true warn_unused_configs = true [[tool.mypy.overrides]] -module = ["pynvml", "pynvml.*", "torch", "torch.*", "triton", "triton.*", "cupy", "cupy.*", "hypothesis", "hypothesis.*"] +module = ["pynvml", "pynvml.*", "torch", "torch.*", "triton", "triton.*", "cupy", "cupy.*", "hypothesis", "hypothesis.*", "psutil", "psutil.*", "tomllib", "tomli"] ignore_missing_imports = true + +# --------------------------------------------------------------------------- +# gpucheck — MPS xfail registry (research SYNTHESIS §2 + §7) +# --------------------------------------------------------------------------- +# +# This block is the LIVING DOCUMENT of known-broken kernels on Apple Silicon +# MPS. It is parsed by `gpucheck.assertions.apply_mps_xfail_config` and +# exposed via `gpucheck.is_mps_xfailed("op.subcategory")`. Tolerance +# multipliers cannot rescue these failures — they are silent-correctness or +# crash bugs in PyTorch's MPS backend. +# +# Re-mine the issue tracker each minor release; bugs that close should be +# removed, and new ones added. +[tool.gpucheck.mps.xfail] +ops = [ + # SDPA correctness on large B×S — pytorch#179352 + "scaled_dot_product_attention.large", + # SDPA backward goes through the math-decomposition backend — pytorch#179294 + "scaled_dot_product_attention.backward", + # layer_norm backward at shape (1,) — pytorch#173525 + "layer_norm.backward.shape1", + # BatchNorm2d backward, channels_last input, ~7-OOM-wrong grads — pytorch#175189 + "batch_norm.backward.channels_last", + # conv2d C_out > 65536 returns zeros — pytorch#142836 + "conv2d.large_channels", + # conv2d backward returns wrong memory format — pytorch#174269 + "conv2d.backward.channels_last_format", + # F.linear backward, BF16/FP16, no-bias, >2D, run-to-run divergence on M5 — pytorch#181936 + "F.linear.backward.bf16_3d_nobias_m5", + # softmax NaN at >10000 in last 2 dims — pytorch#96602 + "softmax.large_attention", + # AvgPool2d backward, channels_last, SIGABRT — pytorch#175190 + "avg_pool2d.backward.channels_last", + # Binary ops on uint16/uint32/uint64 return garbage — pytorch#176296 + "binary_ops.uint16_uint32_uint64", + # BCE loss broken since 2024 — pytorch#137001 + "BCE_loss", + # Catastrophic gradient corruption when total elements > 32K — pytorch#177116 + "matmul.backward.over_32K_elements", +] + +[tool.mutmut] +paths_to_mutate = "src/gpucheck/" +runner = "uv run pytest -x -q" +tests_dir = "tests/" diff --git a/src/gpucheck/__init__.py b/src/gpucheck/__init__.py index 078fb48..5f2b18f 100644 --- a/src/gpucheck/__init__.py +++ b/src/gpucheck/__init__.py @@ -5,13 +5,16 @@ import importlib from typing import TYPE_CHECKING, Any -__version__ = "0.1.0" +__version__ = "1.0.0rc1" _LAZY_MAP: dict[str, tuple[str, str]] = { "assert_close": ("gpucheck.assertions", "assert_close"), "compute_tolerance": ("gpucheck.assertions", "compute_tolerance"), "tolerance_context": ("gpucheck.assertions", "tolerance_context"), + "is_mps_xfailed": ("gpucheck.assertions", "is_mps_xfailed"), + "mps_xfail_list": ("gpucheck.assertions", "mps_xfail_list"), + "register_mps_xfail": ("gpucheck.assertions", "register_mps_xfail"), "dtypes": ("gpucheck.decorators", "dtypes"), "shapes": ("gpucheck.decorators", "shapes"), "devices": ("gpucheck.decorators", "devices"), @@ -31,6 +34,9 @@ "gpu_count": ("gpucheck.arch", "gpu_count"), "BenchmarkResult": ("gpucheck.fixtures.benchmark", "BenchmarkResult"), "GPUDevice": ("gpucheck.fixtures.gpu", "GPUDevice"), + "available_backends": ("gpucheck.backends", "available_backends"), + "get_backend": ("gpucheck.backends", "get_backend"), + "Backend": ("gpucheck.backends", "Backend"), } @@ -69,6 +75,11 @@ def __getattr__(name: str) -> Any: __all__ = [ "__version__", "assert_close", + "compute_tolerance", + "tolerance_context", + "is_mps_xfailed", + "mps_xfail_list", + "register_mps_xfail", "dtypes", "shapes", "devices", @@ -80,6 +91,9 @@ def __getattr__(name: str) -> Any: "gpu_count", "BenchmarkResult", "GPUDevice", + "available_backends", + "get_backend", + "Backend", "FLOAT_DTYPES", "HALF_DTYPES", "ALL_DTYPES", diff --git a/src/gpucheck/arch/__init__.py b/src/gpucheck/arch/__init__.py index 56a912f..7d66c53 100644 --- a/src/gpucheck/arch/__init__.py +++ b/src/gpucheck/arch/__init__.py @@ -2,7 +2,11 @@ from __future__ import annotations -from gpucheck.arch.compatibility import require_arch, require_capability +from gpucheck.arch.compatibility import ( + require_arch, + require_capability, + requires_arch, +) from gpucheck.arch.detection import GPUInfo, detect_gpus from gpucheck.arch.tensor_cores import supports_tensor_cores, warn_tensor_core_fallback @@ -29,8 +33,13 @@ def detect_gpu() -> GPUInfo | None: "detect_gpus", "gpu_available", "gpu_count", + # @requires_arch is the canonical (plural) form, consistent with + # @requires_determinism. @require_arch is the deprecated singular + # alias kept for v1.0 backward compatibility; it emits a + # DeprecationWarning and will be removed in v1.2. "require_arch", "require_capability", + "requires_arch", "supports_tensor_cores", "warn_tensor_core_fallback", ] diff --git a/src/gpucheck/arch/compatibility.py b/src/gpucheck/arch/compatibility.py index 93c4c4c..c9ad803 100644 --- a/src/gpucheck/arch/compatibility.py +++ b/src/gpucheck/arch/compatibility.py @@ -55,12 +55,16 @@ def _get_primary_gpu() -> GPUInfo | None: return gpus[0] if gpus else None -def require_arch(*archs: str) -> Callable[..., Any]: +def requires_arch(*archs: str) -> Callable[..., Any]: """Decorator: skip test if GPU architecture doesn't match any of the given names. + Canonical (plural) name. The singular ``require_arch`` is a + deprecated alias kept for v1.0 backward compatibility; it will be + removed in v1.2. + Usage:: - @require_arch("Ampere", "Hopper") + @requires_arch("Ampere", "Hopper") def test_something(): ... """ @@ -92,6 +96,30 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: return decorator +def require_arch(*archs: str) -> Callable[..., Any]: + """Deprecated singular alias of :func:`requires_arch`. + + The original v1.0 spelling. Inconsistent with the plural + ``requires_determinism``; v1.1 introduces ``requires_arch`` as the + canonical name and v1.2 will remove this singular form. Calling this + decorator factory emits a :class:`DeprecationWarning`. + + Usage (deprecated, use ``requires_arch`` instead):: + + @require_arch("Ampere", "Hopper") + def test_something(): + ... + """ + warnings.warn( + "@require_arch (singular) is deprecated and will be removed in " + "v1.2; use @requires_arch (plural) for naming consistency with " + "@requires_determinism.", + DeprecationWarning, + stacklevel=2, + ) + return requires_arch(*archs) + + def require_capability(major: int, minor: int = 0) -> Callable[..., Any]: """Decorator: skip test if GPU compute capability is below (major, minor). diff --git a/src/gpucheck/arch/detection.py b/src/gpucheck/arch/detection.py index 657af63..f4be2d7 100644 --- a/src/gpucheck/arch/detection.py +++ b/src/gpucheck/arch/detection.py @@ -103,7 +103,15 @@ def _tensor_core_gen(cc: tuple[int, int], name: str = "") -> int | None: @dataclass(frozen=True, slots=True) class GPUInfo: - """Detailed information about a single GPU device.""" + """Detailed information about a single GPU device. + + The ``backend`` field disambiguates CUDA vs MPS GPUs. Defaults to + ``"cuda"`` so existing pre-v1.0 callers (and tests) keep working + unchanged. MPS-derived ``GPUInfo`` instances populate + ``compute_capability=(0, 0)``, ``cuda_version=""``, + ``tensor_core_generation=None``, ``supports_fp8=False``, + ``supports_tf32=False`` and use ``architecture="Apple-Silicon"``. + """ device_id: int name: str @@ -119,6 +127,7 @@ class GPUInfo: supports_tf32: bool tensor_core_generation: int | None max_shared_memory_per_block: int # bytes + backend: str = "cuda" # "cuda" | "mps" def _detect_via_pynvml() -> list[GPUInfo] | None: @@ -145,7 +154,8 @@ def _detect_via_pynvml() -> list[GPUInfo] | None: cuda_major = cuda_ver_int // 1000 cuda_minor = (cuda_ver_int % 1000) // 10 cuda_version = f"{cuda_major}.{cuda_minor}" - except Exception: + except (pynvml.NVMLError, AttributeError) as exc: + logger.debug("pynvml CUDA driver version probe failed: %s", exc) cuda_version = "" device_count = pynvml.nvmlDeviceGetCount() @@ -217,7 +227,8 @@ def _detect_via_torch() -> list[GPUInfo] | None: torch.cuda.set_device(i) free_bytes, _total = torch.cuda.mem_get_info(i) free_mb = free_bytes // (1024 * 1024) - except Exception: + except (RuntimeError, AttributeError) as exc: + logger.debug("torch.cuda.mem_get_info(%d) failed: %s; using total as free", i, exc) free_mb = total_mb # best guess if hasattr(props, "max_shared_memory_per_block"): @@ -256,13 +267,17 @@ def _default_shared_memory(cc: tuple[int, int]) -> int: return 48 * 1024 # pre-Volta -@lru_cache(maxsize=1) -def detect_gpus() -> list[GPUInfo]: - """Detect all available GPUs and return their info. +def _detect_gpus_or_warn() -> list[GPUInfo] | None: + """Single source of truth for GPU detection (T-20). - Uses pynvml as the primary backend (no torch import needed). - Falls back to torch.cuda if pynvml is unavailable. - Result is cached for the session lifetime. + Tries pynvml first, then ``torch.cuda``; returns the first non-None + result. Returns ``None`` only when no detection backend is importable — + in that case a one-shot UserWarning is emitted so callers in either + ``arch/detection.detect_gpus`` or ``fixtures/gpu.detect_gpu`` can + decide whether to map ``None`` to an empty list or to ``None``. + + A backend that imports but reports zero GPUs returns ``[]`` (truthy + Python-False but distinct from ``None``). """ gpus = _detect_via_pynvml() if gpus is not None: @@ -280,4 +295,16 @@ def detect_gpus() -> list[GPUInfo]: "No GPU detection backend available. Install pynvml or torch for GPU support.", stacklevel=2, ) - return [] + return None + + +@lru_cache(maxsize=1) +def detect_gpus() -> list[GPUInfo]: + """Detect all available GPUs and return their info. + + Uses pynvml as the primary backend (no torch import needed). + Falls back to torch.cuda if pynvml is unavailable. + Result is cached for the session lifetime. + """ + gpus = _detect_gpus_or_warn() + return [] if gpus is None else gpus diff --git a/src/gpucheck/assertions/__init__.py b/src/gpucheck/assertions/__init__.py index 1b2ed77..3d6faff 100644 --- a/src/gpucheck/assertions/__init__.py +++ b/src/gpucheck/assertions/__init__.py @@ -3,6 +3,23 @@ from __future__ import annotations from gpucheck.assertions.close import assert_close -from gpucheck.assertions.tolerances import compute_tolerance, tolerance_context +from gpucheck.assertions.tolerances import ( + apply_mps_xfail_config, + compute_tolerance, + is_mps_xfailed, + mps_xfail_list, + register_mps_xfail, + reset_mps_xfail, + tolerance_context, +) -__all__ = ["assert_close", "compute_tolerance", "tolerance_context"] +__all__ = [ + "assert_close", + "compute_tolerance", + "tolerance_context", + "is_mps_xfailed", + "mps_xfail_list", + "apply_mps_xfail_config", + "register_mps_xfail", + "reset_mps_xfail", +] diff --git a/src/gpucheck/assertions/close.py b/src/gpucheck/assertions/close.py index 97ee190..448452c 100644 --- a/src/gpucheck/assertions/close.py +++ b/src/gpucheck/assertions/close.py @@ -10,13 +10,29 @@ from gpucheck.assertions.reporting import format_mismatch_report from gpucheck.assertions.tolerances import compute_tolerance -try: - import torch as _torch +# Cached lazy-imported torch module (None if not installed). +# Module-level cache; sentinel `_TORCH_UNRESOLVED` distinguishes "not yet looked +# up" from "looked up and absent" so the lookup happens at most once. +_TORCH_UNRESOLVED: Any = object() +_torch_cached: Any = _TORCH_UNRESOLVED - _has_torch = True -except ImportError: - _torch = None # type: ignore[assignment] - _has_torch = False + +def _torch_mod() -> Any: + """Lazy-import torch and cache the module (or ``None`` if unavailable). + + CLAUDE.md mandates that torch is never imported at collection / import + time — only when an API actually needs it. This helper is the single + point of access; every call site goes through it. + """ + global _torch_cached + if _torch_cached is _TORCH_UNRESOLVED: + try: + import torch as _t + except ImportError: + _torch_cached = None + else: + _torch_cached = _t + return _torch_cached def _to_numpy(tensor: Any) -> npt.NDArray[Any]: @@ -26,7 +42,10 @@ def _to_numpy(tensor: Any) -> npt.NDArray[Any]: # torch.Tensor if hasattr(tensor, "detach"): - t = tensor.detach().cpu() + # `.contiguous()` is required on torch <2.1 to avoid RuntimeError on + # stride-fuzzed / sliced / transposed inputs when calling `.numpy()`. + # Preventive on newer torch — known to fire on older PyTorch (PM-4). + t = tensor.detach().cpu().contiguous() # Preserve float64 precision; only cast non-numpy-compatible dtypes if t.is_floating_point(): if t.dtype.itemsize >= 8: @@ -53,7 +72,7 @@ def _to_numpy(tensor: Any) -> npt.NDArray[Any]: try: import torch - t = torch.as_tensor(tensor).detach().cpu() + t = torch.as_tensor(tensor).detach().cpu().contiguous() if t.is_floating_point(): if t.dtype.itemsize >= 8: return t.double().numpy() @@ -139,33 +158,40 @@ def assert_close( """ dtype = _resolve_dtype(actual, expected) + # --- Resolve device type so MPS gets the PROVISIONAL 2x tolerance overlay --- + # SYNTHESIS §7: MPS multipliers are PROVISIONAL until calibrated on + # M-silicon; numbers may inflate post-calibration. + device_type: str | None = None + _torch = _torch_mod() + if _torch is not None: + for t in (actual, expected): + if isinstance(t, _torch.Tensor): + device_type = t.device.type + break + # --- Compute effective tolerances up-front (needed by both paths) --- - if baseline_2x and atol is None and rtol is None: - # FlashAttention 2x: double base tolerance BEFORE k_dim scaling - base_atol, base_rtol = compute_tolerance(dtype) - doubled_atol, doubled_rtol = base_atol * 2.0, base_rtol * 2.0 - # Now apply k_dim scaling on the doubled base - if k_dim is not None and k_dim > 0: - import math - - doubled_atol *= math.sqrt(k_dim) - eff_atol = doubled_atol - eff_rtol = doubled_rtol - else: - default_atol, default_rtol = compute_tolerance(dtype, k_dim=k_dim) - eff_atol = atol if atol is not None else default_atol - eff_rtol = rtol if rtol is not None else default_rtol - if baseline_2x: - eff_atol *= 2.0 - eff_rtol *= 2.0 + # Canonical k_dim scaling lives in :func:`compute_tolerance` + # (``sqrt(max(k_dim, 1) / 128)``). The baseline_2x knob multiplies the + # canonical-scaled tolerance by 2 — it must NOT re-derive its own + # k_dim scale, or the two code paths diverge by a factor of + # ``sqrt(128) ≈ 11x`` at large k_dim (review BLOCKER N1). + base_atol, base_rtol = compute_tolerance( + dtype, k_dim=k_dim, device_type=device_type, + ) + eff_atol = atol if atol is not None else base_atol + eff_rtol = rtol if rtol is not None else base_rtol + if baseline_2x: + eff_atol *= 2.0 + eff_rtol *= 2.0 # --- GPU fast-path: avoid CPU transfer when tensors match --- + # Widened to MPS in v1.0; torch.allclose is device-agnostic. if ( - _has_torch + _torch is not None and isinstance(actual, _torch.Tensor) and isinstance(expected, _torch.Tensor) and actual.device == expected.device - and actual.device.type == "cuda" + and actual.device.type in ("cuda", "mps") and actual.shape == expected.shape ): try: @@ -173,7 +199,7 @@ def assert_close( return # PASS — no CPU transfer needed except RuntimeError as exc: if "allclose" not in str(exc).lower() and "match" not in str(exc).lower(): - raise # Re-raise genuine CUDA errors + raise # Re-raise genuine CUDA / MPS errors # --- Slow path: rich error reporting via numpy --- actual_np = _to_numpy(actual) diff --git a/src/gpucheck/assertions/tolerances.py b/src/gpucheck/assertions/tolerances.py index c192a26..bef61c3 100644 --- a/src/gpucheck/assertions/tolerances.py +++ b/src/gpucheck/assertions/tolerances.py @@ -4,6 +4,7 @@ import math from contextlib import contextmanager +from contextvars import ContextVar from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -11,7 +12,8 @@ _DEFAULT_TOLERANCES: dict[str, tuple[float, float]] = { # dtype_name: (atol, rtol) - # Calibrated against cuBLAS matmul on Turing/Ampere GPUs. + # Calibrated against cuBLAS matmul on Turing/Ampere/Ada NVIDIA GPUs + # (see assertions/tolerances and arch/tensor_cores). # atol covers element-wise ops; rtol covers matmul-like ops where # output magnitude scales with input size. "float64": (1e-10, 1e-7), @@ -23,9 +25,45 @@ "tf32": (5e-4, 5e-4), } -# Override stack (module-level). NOT thread-safe — each thread/worker should use -# its own process (pytest-xdist worker) for parallel test execution. -_tolerance_overrides: list[tuple[float, float]] = [] +# PROVISIONAL — research SYNTHESIS §7. These multipliers are mapped from the +# real PyTorch MPS bug magnitudes documented in +# `.claude/teams/research/v1.0/SYNTHESIS.md` (pytorch#177116, #181936, #178497, +# #142836, #173525, #175189, #96602 etc.) but the precise values must be +# calibrated on Akash's actual M-generation hardware before being canonical +# (sub-Q 7 § "Calibration plan"). Until then, treat as a directional overlay. +# 2× is the FlashAttention precedent (assertions/close.py:117 baseline_2x). +# +# v1.1 calibration data on Apple M5 (5K samples × 21 cells, 2026-05-07) is +# captured at `.claude/teams/audit/v1.1/drift_histogram_5k.json` and analysed +# in `.claude/teams/audit/v1.1/EVIDENCE/calibration-final.md`. Headline: +# matmul cells confirm the 2× starting point within ±6% (need 13.4×/17.7×/27.6× +# for fp32/fp16/bf16 P99 — already exceeded by the FA precedent in many cases), +# but conv2d is a real outlier (+225% fp32, +75% fp16, +70% bf16 vs v3 200-iter +# projection). The per-(kernel, dtype) refactor is task T-24 in +# IMPLEMENTATION_PLAN_v1.1.md and ships in v1.1, not v1.0. +_MPS_TOLERANCE_MULTIPLIERS: dict[str, float] = { + "float32": 2.0, + "float16": 2.0, + "bfloat16": 2.0, + "float64": 1.0, # Rarely load-bearing on MPS; keep CUDA tolerance. + "float8_e4m3fn": 2.0, # Apple Silicon has no FP8 tensor cores; placeholder. + "float8_e5m2": 2.0, + "tf32": 1.0, # TF32 is NVIDIA-only; Apple Silicon has no analogue. +} + +# Override stack (per-context). Backed by ``contextvars.ContextVar`` so the +# stack is isolated per OS thread AND per asyncio task. Previous releases +# used a plain module-level list, which leaked overrides between threads +# when tests were run inside a single process. ContextVar.set returns a +# Token that ``ContextVar.reset`` consumes, restoring the prior value — +# correct under exception unwinding. +_tolerance_overrides: ContextVar[tuple[tuple[float, float], ...]] = ContextVar( + "_tolerance_overrides", default=(), +) + +# MPS xfail registry — populated by `apply_mps_xfail_config` from +# ``[tool.gpucheck.mps.xfail]``. Tests can query via :func:`is_mps_xfailed`. +_mps_xfail_set: set[str] = set() def _normalize_dtype_name(dtype: Any) -> str: @@ -42,6 +80,7 @@ def compute_tolerance( dtype: Any, *, k_dim: int | None = None, + device_type: str | None = None, ) -> tuple[float, float]: """Return (atol, rtol) for a given dtype. @@ -50,11 +89,17 @@ def compute_tolerance( model where 128 is the standard tile dimension. This means at k_dim=128 the tolerance is 1x the base, and scales proportionally from there. + If *device_type* is ``"mps"``, an additional dtype-specific multiplier + from :data:`_MPS_TOLERANCE_MULTIPLIERS` is applied. The multipliers are + PROVISIONAL until calibrated on the user's M-generation hardware + (see SYNTHESIS §7 calibration plan). + Falls back to float32 tolerances for unknown dtypes. """ - # Check override stack first. - if _tolerance_overrides: - return _tolerance_overrides[-1] + # Check override stack first (ContextVar for thread/task isolation). + overrides = _tolerance_overrides.get() + if overrides: + return overrides[-1] name = _normalize_dtype_name(dtype) # Check config overlay first, then defaults @@ -66,6 +111,12 @@ def compute_tolerance( if k_dim is not None and k_dim > 0: atol = atol * math.sqrt(max(k_dim, 1) / 128.0) + # MPS overlay (PROVISIONAL — see SYNTHESIS §7 calibration plan). + if device_type == "mps": + multiplier = _MPS_TOLERANCE_MULTIPLIERS.get(name, 2.0) + atol *= multiplier + rtol *= multiplier + return atol, rtol @@ -76,16 +127,21 @@ def tolerance_context( ) -> Generator[None, None, None]: """Temporarily override default tolerances returned by :func:`compute_tolerance`. + Backed by ``contextvars.ContextVar``: the override is visible only to + the current OS thread (and to asyncio tasks that copied the current + context). Sibling threads observe the underlying defaults concurrently. + Usage:: with tolerance_context(atol=1e-3, rtol=1e-3): assert_close(a, b) """ - _tolerance_overrides.append((atol, rtol)) + current = _tolerance_overrides.get() + token = _tolerance_overrides.set(current + ((atol, rtol),)) try: yield finally: - _tolerance_overrides.pop() + _tolerance_overrides.reset(token) def tolerances_from_config(config: dict[str, Any]) -> dict[str, tuple[float, float]] | None: @@ -128,3 +184,66 @@ def apply_config_tolerances(config: dict[str, Any]) -> None: def reset_config_tolerances() -> None: """Remove all config-based tolerance overrides.""" _config_overrides.clear() + + +# --------------------------------------------------------------------------- +# MPS xfail registry (research SYNTHESIS §2 + §7) +# --------------------------------------------------------------------------- + +def mps_xfail_from_config(config: dict[str, Any]) -> set[str] | None: + """Parse the MPS xfail list from a ``[tool.gpucheck.mps.xfail]`` block. + + Expected shape:: + + [tool.gpucheck.mps.xfail] + ops = [ + "scaled_dot_product_attention.large", + "softmax.large_attention", + ... + ] + + Returns ``None`` when the section is absent or empty so callers can + distinguish "no MPS config" from "explicit empty list". + """ + section = config.get("tool", {}).get("gpucheck", {}).get("mps", {}).get("xfail") + if not section: + return None + ops = section.get("ops") + if not isinstance(ops, list): + return None + return {str(o) for o in ops} + + +def apply_mps_xfail_config(config: dict[str, Any]) -> None: + """Replace the MPS xfail registry with entries from the config block.""" + parsed = mps_xfail_from_config(config) + if parsed is None: + return + _mps_xfail_set.clear() + _mps_xfail_set.update(parsed) + + +def reset_mps_xfail() -> None: + """Drop all MPS xfail registrations.""" + _mps_xfail_set.clear() + + +def register_mps_xfail(*ops: str) -> None: + """Add one or more op names to the MPS xfail registry (test helper).""" + _mps_xfail_set.update(ops) + + +def is_mps_xfailed(op_name: str) -> bool: + """Return ``True`` if *op_name* is in the MPS xfail registry. + + The registry is populated from ``pyproject.toml`` at session start (see + :func:`apply_mps_xfail_config`). Tests can also push entries at runtime + via :func:`register_mps_xfail`. The match is exact-string; the canonical + naming convention is ``op.subcategory`` (see SYNTHESIS §7). + """ + return op_name in _mps_xfail_set + + +def mps_xfail_list() -> list[str]: + """Return the current MPS xfail list, sorted for stable iteration.""" + return sorted(_mps_xfail_set) diff --git a/src/gpucheck/backends/__init__.py b/src/gpucheck/backends/__init__.py new file mode 100644 index 0000000..92f6134 --- /dev/null +++ b/src/gpucheck/backends/__init__.py @@ -0,0 +1,94 @@ +"""Backend abstraction for gpucheck — CUDA and MPS implementations. + +This package defines a structural :class:`Backend` Protocol that captures the +GPU-specific operations gpucheck needs: + +- ``synchronize`` — block until pending work completes on a device +- ``event_timer`` — context-managed timer that uses the cheapest accurate + primitive available (CUDA events on NVIDIA, wall-clock + device sync on MPS, + see SYNTHESIS §3 / pytorch#162872 for the deadlock context) +- ``mem_stats`` — per-device memory accounting +- ``flush_l2`` — best-effort L2-cache eviction for stable benchmark timings +- ``arch_info`` — populate a :class:`gpucheck.arch.GPUInfo` for the device + +The Protocol is **additive** in v1.0: existing CUDA-only call sites in +``fixtures/benchmark.py``, ``fixtures/profiler.py``, etc. retain their direct +``torch.cuda.*`` calls. New MPS code uses the Protocol so the deadlock-safe +timing path is the **only** path on Apple Silicon. + +Public API (re-exported via ``gpucheck.backends``):: + + from gpucheck.backends import Backend, available_backends, get_backend + + backends = available_backends() # list[Backend], priority order + cuda = get_backend("cuda") # raises if unavailable + mps = get_backend("mps") # raises if unavailable +""" + +from __future__ import annotations + +from gpucheck.backends._protocol import Backend, EventTimer + + +def available_backends() -> list[Backend]: + """Return all currently-available backends in priority order. + + Priority is CUDA → MPS, mirroring PyTorch's own dispatch order. CPU is + intentionally excluded — gpucheck targets accelerators. + """ + backends: list[Backend] = [] + + # CUDA first (typical on Linux/Windows GPU servers) + try: + from gpucheck.backends.cuda import CUDABackend + + cuda = CUDABackend() + if cuda.is_available(): + backends.append(cuda) + except ImportError: + pass + + # MPS second (Apple Silicon) + try: + from gpucheck.backends.mps import MPSBackend + + mps = MPSBackend() + if mps.is_available(): + backends.append(mps) + except ImportError: + pass + + return backends + + +def get_backend(name: str) -> Backend: + """Return the named backend, or raise :class:`RuntimeError` if unavailable. + + Recognized names: ``"cuda"``, ``"mps"``. + """ + name_lower = name.lower() + if name_lower == "cuda": + from gpucheck.backends.cuda import CUDABackend + + b: Backend = CUDABackend() + elif name_lower == "mps": + from gpucheck.backends.mps import MPSBackend + + b = MPSBackend() + else: + raise ValueError(f"Unknown backend {name!r}; expected 'cuda' or 'mps'") + + if not b.is_available(): + raise RuntimeError( + f"Backend {name!r} is not available on this system " + f"(missing torch, missing hardware, or driver issue)" + ) + return b + + +__all__ = [ + "Backend", + "EventTimer", + "available_backends", + "get_backend", +] diff --git a/src/gpucheck/backends/_protocol.py b/src/gpucheck/backends/_protocol.py new file mode 100644 index 0000000..087ae56 --- /dev/null +++ b/src/gpucheck/backends/_protocol.py @@ -0,0 +1,76 @@ +"""Backend and EventTimer Protocols. + +Kept in a private module so user code imports from ``gpucheck.backends`` +(public surface) rather than ``gpucheck.backends._protocol``. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable + +if TYPE_CHECKING: + from contextlib import AbstractContextManager + + from gpucheck.arch.detection import GPUInfo + + +@runtime_checkable +class EventTimer(Protocol): + """A single benchmark interval with millisecond elapsed time. + + Concrete instances are produced by :meth:`Backend.event_timer` and used as + context managers:: + + with backend.event_timer() as t: + kernel(x, y) + elapsed_ms = t.elapsed_ms + """ + + @property + def elapsed_ms(self) -> float: + """Elapsed wall time of the protected block in milliseconds.""" + ... + + +@runtime_checkable +class Backend(Protocol): + """Structural interface for a GPU backend supported by gpucheck.""" + + name: str # "cuda" | "mps" + + def is_available(self) -> bool: + """Return ``True`` if this backend can run kernels on this machine.""" + ... + + def device_count(self) -> int: + """Number of devices this backend exposes.""" + ... + + def synchronize(self, device_id: int = 0) -> None: + """Block until pending work on the device has finished.""" + ... + + def event_timer( + self, device_id: int = 0, + ) -> AbstractContextManager[EventTimer]: + """Return a context manager that times the wrapped block.""" + ... + + def mem_stats(self, device_id: int = 0) -> dict[str, int]: + """Memory accounting in bytes; keys at minimum: ``used``, ``total``.""" + ... + + def flush_l2(self, device_id: int = 0, buf: Any = None) -> None: + """Best-effort L2-cache flush for stable benchmark timings. + + On backends without L2-flush support (e.g. MPS), this is a no-op and + emits a one-time :class:`UserWarning`. + """ + ... + + def arch_info(self, device_id: int = 0) -> GPUInfo: + """Populate a :class:`GPUInfo` describing the device.""" + ... + + +__all__ = ["Backend", "EventTimer"] diff --git a/src/gpucheck/backends/cuda.py b/src/gpucheck/backends/cuda.py new file mode 100644 index 0000000..0c45eb1 --- /dev/null +++ b/src/gpucheck/backends/cuda.py @@ -0,0 +1,112 @@ +"""CUDA backend conforming to the gpucheck :class:`Backend` Protocol. + +This is a thin facade over the existing ``torch.cuda.*`` and ``pynvml`` +helpers used elsewhere in the project. Existing call sites in +``fixtures/benchmark.py``, ``fixtures/profiler.py``, ``arch/detection.py`` +keep their direct ``torch.cuda.*`` invocations for v1.0 — this module exists +so that **new** code (especially MPS-aware test code) can write +backend-agnostic loops:: + + backend = get_backend("cuda") # or "mps" + with backend.event_timer() as t: + kernel(x, y) + elapsed = t.elapsed_ms + +A v1.1 refactor will migrate the legacy call sites to consume this Protocol. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Generator + + from gpucheck.arch.detection import GPUInfo + + +def _torch() -> Any: + """Lazy-import torch so the package stays importable without it.""" + import torch + + return torch + + +@dataclass +class _CUDAEventTimer: + """EventTimer backed by ``torch.cuda.Event(enable_timing=True)``.""" + + device_id: int = 0 + elapsed_ms: float = field(default=0.0) + + +class CUDABackend: + """Backend implementation targeting NVIDIA GPUs via ``torch.cuda``.""" + + name: str = "cuda" + + def is_available(self) -> bool: + try: + torch = _torch() + except ImportError: + return False + return bool(torch.cuda.is_available()) + + def device_count(self) -> int: + if not self.is_available(): + return 0 + return int(_torch().cuda.device_count()) + + def synchronize(self, device_id: int = 0) -> None: + torch = _torch() + torch.cuda.synchronize(device_id) + + @contextmanager + def event_timer( + self, device_id: int = 0, + ) -> Generator[_CUDAEventTimer, None, None]: + torch = _torch() + timer = _CUDAEventTimer(device_id=device_id) + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + try: + yield timer + finally: + end.record() + torch.cuda.synchronize(device_id) + timer.elapsed_ms = float(start.elapsed_time(end)) + + def mem_stats(self, device_id: int = 0) -> dict[str, int]: + torch = _torch() + try: + free, total = torch.cuda.mem_get_info(device_id) + except RuntimeError: + return {"used": 0, "total": 0, "free": 0} + used = total - free + return {"used": int(used), "total": int(total), "free": int(free)} + + def flush_l2(self, device_id: int = 0, buf: Any = None) -> None: + # Use existing helper to keep behavior identical. + from gpucheck.fixtures.benchmark import _flush_l2_cache, _get_l2_cache_size + + size = _get_l2_cache_size() + if size <= 0: + return + _flush_l2_cache(size, buf=buf) + + def arch_info(self, device_id: int = 0) -> GPUInfo: + from gpucheck.arch.detection import detect_gpus + + gpus = detect_gpus() + if not gpus or device_id >= len(gpus): + raise RuntimeError( + f"CUDA backend reports no GPU at index {device_id} " + f"(detected {len(gpus)})" + ) + return gpus[device_id] + + +__all__ = ["CUDABackend"] diff --git a/src/gpucheck/backends/mps.py b/src/gpucheck/backends/mps.py new file mode 100644 index 0000000..3cb468b --- /dev/null +++ b/src/gpucheck/backends/mps.py @@ -0,0 +1,245 @@ +"""MPS backend for Apple Silicon GPUs via ``torch.mps.*``. + +# Deadlock context (load-bearing) + +PyTorch issue [pytorch#162872](https://github.com/pytorch/pytorch/issues/162872) +documents a hang in the canonical CUDA-style timing pattern on +Apple Silicon:: + + start = torch.mps.event.Event(enable_timing=True) + end = torch.mps.event.Event(enable_timing=True) + start.record(); kernel(); end.record() + end.synchronize() # <- HANGS on PyTorch 2.10+ Apple Silicon + elapsed = start.elapsed_time(end) + +gpucheck v1.0 therefore times MPS work with **device-level** +``torch.mps.synchronize()`` plus ``time.perf_counter()``. This is correct per +the PyTorch 2.11 docs (verified in research SYNTHESIS §3) and avoids the +deadlock. The ~1ms overhead vs CUDA events is acceptable — gpucheck reports +millisecond-resolution timings, not microsecond. + +# Memory accounting + +PyTorch issue +[pytorch#164299](https://github.com/pytorch/pytorch/issues/164299) notes that +``torch.mps.current_allocated_memory()`` and +``torch.mps.driver_allocated_memory()`` lag Activity Monitor for some +allocation patterns. This MPSBackend therefore returns BOTH numbers (so the +caller can pick) and adds an optional ``rss`` key sourced from ``psutil`` if +that package is importable. ``rss`` is the most accurate leak proxy on MPS. + +# Tolerances and xfail + +This module does **not** carry MPS tolerance multipliers — those live in +``gpucheck.assertions.tolerances`` so that all dtype-aware tolerance logic +shares a single source of truth. +""" + +from __future__ import annotations + +import logging +import platform +import subprocess +import time +import warnings +from contextlib import contextmanager +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Generator + + from gpucheck.arch.detection import GPUInfo + +logger = logging.getLogger(__name__) + + +def _torch() -> Any: + import torch + + return torch + + +_FLUSH_L2_WARNED = False + + +@dataclass +class _MPSEventTimer: + """EventTimer using device-level sync + wall-clock time. + + Avoids ``torch.mps.event.Event.synchronize()`` per pytorch#162872. + """ + + device_id: int = 0 + elapsed_ms: float = field(default=0.0) + + +class MPSBackend: + """Backend implementation for Apple Silicon GPUs.""" + + name: str = "mps" + + def is_available(self) -> bool: + try: + torch = _torch() + except ImportError: + return False + return bool( + getattr(torch.backends, "mps", None) is not None + and torch.backends.mps.is_available() + ) + + def device_count(self) -> int: + if not self.is_available(): + return 0 + # PyTorch's MPS device API only exposes a single logical device. + # torch.mps.device_count() exists on 2.6+, fall back to 1. + torch = _torch() + fn = getattr(torch.mps, "device_count", None) + if callable(fn): + try: + return int(fn()) + except (RuntimeError, AttributeError) as exc: + logger.debug("torch.mps.device_count() failed: %s; falling back to 1", exc) + return 1 + return 1 + + def synchronize(self, device_id: int = 0) -> None: + # device_id is ignored — MPS exposes one logical device. + del device_id + torch = _torch() + # Device-level sync; safe per pytorch#162872. + torch.mps.synchronize() + + @contextmanager + def event_timer( + self, device_id: int = 0, + ) -> Generator[_MPSEventTimer, None, None]: + """Time a block with device-level sync + wall clock. + + DO NOT use ``torch.mps.event.Event.synchronize()`` here: + pytorch#162872 deadlocks the calling thread. + """ + timer = _MPSEventTimer(device_id=device_id) + torch = _torch() + # Drain any prior in-flight work so its time isn't counted in ours. + torch.mps.synchronize() + t0 = time.perf_counter() + try: + yield timer + finally: + # Block until the kernel(s) launched in the body actually finish. + torch.mps.synchronize() + timer.elapsed_ms = (time.perf_counter() - t0) * 1000.0 + + def mem_stats(self, device_id: int = 0) -> dict[str, int]: + del device_id # MPS = single device + torch = _torch() + stats: dict[str, int] = {} + try: + stats["used"] = int(torch.mps.current_allocated_memory()) + except (RuntimeError, AttributeError) as exc: + logger.debug("torch.mps.current_allocated_memory() failed: %s", exc) + stats["used"] = 0 + try: + stats["driver_allocated"] = int(torch.mps.driver_allocated_memory()) + except (RuntimeError, AttributeError) as exc: + logger.debug("torch.mps.driver_allocated_memory() failed: %s", exc) + stats["driver_allocated"] = 0 + # recommended_max_memory exists on 2.6+ + rec_fn = getattr(torch.mps, "recommended_max_memory", None) + if callable(rec_fn): + try: + stats["total"] = int(rec_fn()) + except (RuntimeError, AttributeError) as exc: + logger.debug("torch.mps.recommended_max_memory() failed: %s", exc) + stats["total"] = 0 + else: + stats["total"] = 0 + # psutil RSS — best leak proxy per pytorch#164299 + try: + import psutil + + stats["rss"] = int(psutil.Process().memory_info().rss) + except ImportError: + pass + return stats + + def flush_l2(self, device_id: int = 0, buf: Any = None) -> None: + """MPS does not expose an L2-cache flush primitive; this is a no-op. + + Emits a one-time :class:`UserWarning` so callers know their + ``flush_l2=True`` request was ignored. + """ + global _FLUSH_L2_WARNED # noqa: PLW0603 + del device_id, buf + if not _FLUSH_L2_WARNED: + warnings.warn( + "MPS backend does not implement L2 cache flush; " + "benchmark stability may be lower than on CUDA", + UserWarning, + stacklevel=2, + ) + _FLUSH_L2_WARNED = True + + def arch_info(self, device_id: int = 0) -> GPUInfo: + del device_id + from gpucheck.arch.detection import GPUInfo + + chip = _detect_apple_chip() + os_ver = platform.mac_ver()[0] or "" + # Memory total: prefer recommended_max_memory; fall back to RSS-zero. + torch = _torch() + rec_fn = getattr(torch.mps, "recommended_max_memory", None) + if callable(rec_fn): + try: + total_bytes = int(rec_fn()) + except (RuntimeError, AttributeError) as exc: + logger.debug( + "torch.mps.recommended_max_memory() failed in arch_info: %s", exc, + ) + total_bytes = 0 + else: + total_bytes = 0 + free_bytes = max(0, total_bytes - int( + getattr(torch.mps, "current_allocated_memory", lambda: 0)() + )) + + return GPUInfo( + device_id=0, + name=chip or "Apple Silicon", + compute_capability=(0, 0), + architecture="Apple-Silicon", + memory_total_mb=total_bytes // (1024 * 1024), + memory_free_mb=free_bytes // (1024 * 1024), + driver_version=os_ver, + cuda_version="", + supports_fp16=True, + supports_bf16=True, + supports_fp8=False, # No FP8 tensor cores on Apple Silicon as of M5 + supports_tf32=False, # TF32 is NVIDIA-only + tensor_core_generation=None, # Apple GPUs have no tensor cores + max_shared_memory_per_block=32 * 1024, # Apple GPU threadgroup memory cap (typical) + backend="mps", + ) + + +def _detect_apple_chip() -> str: + """Return e.g. ``"Apple M4 Pro"`` or empty string on failure. + + Uses ``sysctl machdep.cpu.brand_string``; we deliberately do **not** call + ``xcrun metal`` (security finding N1) and do **not** read ``task_info`` + (N3) — both are out-of-scope per CHARTER waivers. + """ + try: + out = subprocess.check_output( + ["sysctl", "-n", "machdep.cpu.brand_string"], + text=True, + timeout=2.0, + ) + return out.strip() + except (FileNotFoundError, subprocess.CalledProcessError, subprocess.TimeoutExpired): + return "" + + +__all__ = ["MPSBackend"] diff --git a/src/gpucheck/decorators/devices.py b/src/gpucheck/decorators/devices.py index 8a14c82..6c8eb6c 100644 --- a/src/gpucheck/decorators/devices.py +++ b/src/gpucheck/decorators/devices.py @@ -1,4 +1,4 @@ -"""Parametrize tests across GPU devices.""" +"""Parametrize tests across GPU devices (CUDA and MPS).""" from __future__ import annotations @@ -22,6 +22,27 @@ def _detect_cuda_devices() -> list[str]: return [] +def _detect_mps_devices() -> list[str]: + """Return ``["mps"]`` if Apple Silicon MPS is available, else ``[]``. + + PyTorch's MPS backend exposes a single logical device, so we never emit + ``mps:0``/``mps:1`` even on machines with an integrated + discrete GPU. + """ + try: + import torch + except ImportError: + return [] + mps = getattr(torch.backends, "mps", None) + if mps is None or not mps.is_available(): + return [] + return ["mps"] + + +def _detect_devices() -> list[str]: + """Return all available accelerator device strings (CUDA first, then MPS).""" + return _detect_cuda_devices() + _detect_mps_devices() + + def _is_device_available(device: str) -> bool: """Check whether a device string is currently usable.""" try: @@ -36,6 +57,9 @@ def _is_device_available(device: str) -> bool: idx = int(device.split(":")[1]) return idx < torch.cuda.device_count() return True + if device == "mps" or device.startswith("mps:"): + mps = getattr(torch.backends, "mps", None) + return bool(mps is not None and mps.is_available()) # Unknown device type — let torch figure it out torch.device(device) return True @@ -55,21 +79,25 @@ def _device_id(d: str) -> str: def devices(*device_args: str) -> Callable[..., Any]: """Parametrize a test across GPU devices. - If no arguments are given, auto-detects all available CUDA devices - (falls back to ``["cuda:0"]`` if detection finds nothing but CUDA - appears importable). + Recognized device strings: + + - ``"cuda:N"`` — specific NVIDIA GPU + - ``"mps"`` — Apple Silicon GPU (single logical device) + - ``"all"`` — every available accelerator (CUDA devices + MPS if present) - Pass ``"all"`` to expand to every visible CUDA device. + If no arguments are given, auto-detects all available accelerators + (CUDA devices first, then MPS). Falls back to ``["cuda:0"]`` if + detection finds nothing — the test then skips at collection. Devices that are not available at collection time get ``pytest.mark.skip`` so the test is reported but not run. Examples:: - @devices("cuda:0", "cuda:1") + @devices("cuda:0", "mps") def test_copy(device): ... - @devices() # auto-detect + @devices() # auto-detect (CUDA + MPS) def test_kernel(device): ... @devices("all") @@ -78,12 +106,12 @@ def test_broadcast(device): ... resolved: list[str] = [] if not device_args or device_args == ("all",): - detected = _detect_cuda_devices() + detected = _detect_devices() resolved = detected if detected else ["cuda:0"] else: for d in device_args: if d == "all": - resolved.extend(_detect_cuda_devices() or ["cuda:0"]) + resolved.extend(_detect_devices() or ["cuda:0"]) else: resolved.append(d) diff --git a/src/gpucheck/decorators/parametrize.py b/src/gpucheck/decorators/parametrize.py index 226a1e7..19052c4 100644 --- a/src/gpucheck/decorators/parametrize.py +++ b/src/gpucheck/decorators/parametrize.py @@ -8,7 +8,7 @@ import pytest -from gpucheck.decorators.devices import _detect_cuda_devices, _is_device_available +from gpucheck.decorators.devices import _detect_devices, _is_device_available from gpucheck.decorators.dtypes import DtypeArg, _dtype_id, _resolve_dtype from gpucheck.decorators.shapes import Shape, _shape_id @@ -16,12 +16,19 @@ SkipFilter = Callable[..., bool] | None -def _combo_id(dtype: Any, shape: Shape, device: str) -> str: - """Build a human-readable test ID: 'float16-128x128-cuda0'.""" +def _combo_id( + dtype: Any, + shape: Shape, + device: str, + stride_category: str | None = None, +) -> str: + """Build a human-readable test ID: 'float16-128x128-cuda0[-broadcast]'.""" parts: list[str] = [] parts.append(_dtype_id(dtype)) parts.append(_shape_id(shape)) parts.append(device.replace(":", "")) + if stride_category is not None: + parts.append(stride_category) return "-".join(parts) @@ -31,6 +38,7 @@ def parametrize_gpu( shapes: Sequence[Shape] = ((128, 128),), devices: Sequence[str] | None = None, skip: SkipFilter = None, + stride_categories: Sequence[str] | None = None, ) -> Callable[..., Any]: """Parametrize a test over the cartesian product of dtypes x shapes x devices. @@ -38,50 +46,113 @@ def parametrize_gpu( dtypes: Dtype strings or torch.dtype objects. shapes: Tensor shape tuples. devices: Device strings. ``None`` auto-detects CUDA devices. - skip: Optional callable ``(dtype, shape, device) -> bool``. - Return ``True`` to skip that combination. - - Example:: + skip: Optional callable ``(dtype, shape, device) -> bool`` + (or ``(dtype, shape, device, stride_category) -> bool`` when + ``stride_categories`` is set). Return ``True`` to skip that + combination. + stride_categories: Optional sequence of stride-fuzzing categories + (see :data:`gpucheck.fuzzing.STRIDE_CATEGORIES`). When + provided, the test signature gains a ``stride_category: str`` + parameter and the cartesian product expands accordingly. Use + :func:`gpucheck.fuzzing.fuzz_strides_for_category` inside the + test body to materialize the perturbed tensor. + + Examples:: @parametrize_gpu( dtypes=("float16", "bfloat16"), shapes=((128, 128), (256, 256)), devices=("cuda:0",), ) - def test_kernel(dtype, shape, device): + def test_kernel(dtype, shape, device): ... + + @parametrize_gpu( + dtypes=("float32",), + shapes=((64, 64),), + stride_categories=("row_major", "transpose", "broadcast"), + ) + def test_layout_invariant(dtype, shape, device, stride_category): + from gpucheck.fuzzing import fuzz_strides_for_category + t = fuzz_strides_for_category(shape, dtype, stride_category, device=device) ... """ # Resolve dtypes resolved_dtypes = [_resolve_dtype(d) for d in dtypes] - # Resolve devices + # Resolve devices: auto-detect CUDA + MPS when caller passes ``None``. if devices is None: - detected = _detect_cuda_devices() + detected = _detect_devices() resolved_devices = detected if detected else ["cuda:0"] else: resolved_devices = list(devices) + # Resolve stride categories + use_strides = stride_categories is not None + resolved_strides: list[str] = list(stride_categories) if stride_categories else [] + if use_strides: + # Validate eagerly; bad input here is a test-author bug. + from gpucheck.fuzzing.strides import CATEGORIES as _ALLOWED + + invalid = [c for c in resolved_strides if c not in _ALLOWED] + if invalid: + raise ValueError( + f"Unknown stride categories: {invalid}; " + f"expected from {sorted(_ALLOWED)}" + ) + # Build cartesian product as pytest.param entries params: list[Any] = [] - for dtype_val, shape_val, dev_val in itertools.product( - resolved_dtypes, shapes, resolved_devices - ): - test_id = _combo_id(dtype_val, shape_val, dev_val) - marks: list[Any] = [] - if skip is not None and skip(dtype_val, shape_val, dev_val): - marks.append(pytest.mark.skip(reason="filtered by skip predicate")) + if not use_strides: + for dtype_val, shape_val, dev_val in itertools.product( + resolved_dtypes, shapes, resolved_devices, + ): + test_id = _combo_id(dtype_val, shape_val, dev_val) + marks: list[Any] = [] + + if skip is not None and skip(dtype_val, shape_val, dev_val): + marks.append(pytest.mark.skip(reason="filtered by skip predicate")) + + if not _is_device_available(dev_val): + marks.append( + pytest.mark.skip(reason=f"device {dev_val} not available"), + ) + + params.append( + pytest.param(dtype_val, shape_val, dev_val, id=test_id, marks=marks), + ) + + return pytest.mark.parametrize("dtype,shape,device", params) + + # Stride-fuzzing branch: cartesian also includes stride_category. + for dtype_val, shape_val, dev_val, stride_cat in itertools.product( + resolved_dtypes, shapes, resolved_devices, resolved_strides, + ): + test_id = _combo_id(dtype_val, shape_val, dev_val, stride_cat) + marks = [] + + if skip is not None: + # 4-arg skip; tolerate 3-arg by checking signature length. + try: + hit = skip(dtype_val, shape_val, dev_val, stride_cat) + except TypeError: + hit = skip(dtype_val, shape_val, dev_val) + if hit: + marks.append(pytest.mark.skip(reason="filtered by skip predicate")) if not _is_device_available(dev_val): marks.append( - pytest.mark.skip(reason=f"device {dev_val} not available") + pytest.mark.skip(reason=f"device {dev_val} not available"), ) params.append( - pytest.param(dtype_val, shape_val, dev_val, id=test_id, marks=marks) + pytest.param( + dtype_val, shape_val, dev_val, stride_cat, + id=test_id, marks=marks, + ), ) - return pytest.mark.parametrize("dtype,shape,device", params) + return pytest.mark.parametrize("dtype,shape,device,stride_category", params) __all__ = ["parametrize_gpu"] diff --git a/src/gpucheck/fixtures/benchmark.py b/src/gpucheck/fixtures/benchmark.py index 7bb4f52..984413a 100644 --- a/src/gpucheck/fixtures/benchmark.py +++ b/src/gpucheck/fixtures/benchmark.py @@ -153,7 +153,17 @@ def __call__( flush_l2: bool | None = None, **kwargs: Any, ) -> BenchmarkResult: - """Benchmark *fn* using CUDA events for accurate GPU timing. + """Benchmark *fn* using accurate GPU timing. + + Backend selection: + + - **CUDA**: ``torch.cuda.Event(enable_timing=True)`` start/end + + ``torch.cuda.synchronize()`` (microsecond resolution). + - **MPS**: ``torch.mps.synchronize()`` (device-level) + + ``time.perf_counter()`` for wall clock. The CUDA-style per-event + ``end.synchronize()`` pattern is deliberately avoided because it + deadlocks on Apple Silicon (pytorch#162872; SYNTHESIS §3). + - **No GPU**: ``pytest.skip``. Parameters ---------- @@ -166,7 +176,9 @@ def __call__( rounds: Override default benchmark iterations. flush_l2: - Override default L2 flushing behaviour. + Override default L2 flushing behaviour. Ignored on MPS (no + L2-flush primitive); a one-time UserWarning is emitted by the + MPS backend. **kwargs: Keyword arguments forwarded to *fn*. """ @@ -174,39 +186,27 @@ def __call__( import torch except ImportError as exc: raise RuntimeError( - "gpu_benchmark requires PyTorch for CUDA event timing. " + "gpu_benchmark requires PyTorch for accurate GPU timing. " "Install it with: pip install torch" ) from exc - if not torch.cuda.is_available(): - pytest.skip("CUDA not available for benchmarking") + cuda_avail = torch.cuda.is_available() + mps_avail = ( + getattr(torch.backends, "mps", None) is not None + and torch.backends.mps.is_available() + ) + + if not cuda_avail and not mps_avail: + pytest.skip("No GPU (CUDA or MPS) available for benchmarking") n_warmup = warmup if warmup is not None else self.warmup n_rounds = rounds if rounds is not None else self.rounds do_flush = flush_l2 if flush_l2 is not None else self.flush_l2 - # Warmup - for _ in range(n_warmup): - fn(*args, **kwargs) - torch.cuda.synchronize() - - # Pre-allocate CUDA events to avoid per-iteration allocation overhead - start = torch.cuda.Event(enable_timing=True) # type: ignore[no-untyped-call] - end = torch.cuda.Event(enable_timing=True) # type: ignore[no-untyped-call] - - # Timed runs - raw_times: list[float] = [] - for _ in range(n_rounds): - if do_flush and self._l2_size > 0: - _flush_l2_cache(self._l2_size, buf=self._flush_buf) - - start.record() - fn(*args, **kwargs) - end.record() - - torch.cuda.synchronize() - elapsed_ms: float = start.elapsed_time(end) - raw_times.append(elapsed_ms) + if cuda_avail: + raw_times = self._run_cuda(fn, args, kwargs, n_warmup, n_rounds, do_flush) + else: + raw_times = self._run_mps(fn, args, kwargs, n_warmup, n_rounds, do_flush) # Outlier removal cleaned = _remove_outliers_iqr(raw_times) @@ -242,10 +242,97 @@ def __call__( raw_times=tuple(raw_times), ) + # ------------------------------------------------------------------ + # Backend-specific timing loops + # ------------------------------------------------------------------ + + def _run_cuda( + self, + fn: KernelCallable, + args: tuple[Any, ...], + kwargs: dict[str, Any], + n_warmup: int, + n_rounds: int, + do_flush: bool, + ) -> list[float]: + """CUDA-events timing loop (microsecond accurate via cudaEvent_t).""" + import torch + + for _ in range(n_warmup): + fn(*args, **kwargs) + torch.cuda.synchronize() + + # Pre-allocate CUDA events to avoid per-iteration allocation overhead. + start = torch.cuda.Event(enable_timing=True) # type: ignore[no-untyped-call] + end = torch.cuda.Event(enable_timing=True) # type: ignore[no-untyped-call] + + raw_times: list[float] = [] + for _ in range(n_rounds): + if do_flush and self._l2_size > 0: + _flush_l2_cache(self._l2_size, buf=self._flush_buf) + + start.record() + fn(*args, **kwargs) + end.record() + + torch.cuda.synchronize() + elapsed_ms: float = start.elapsed_time(end) + raw_times.append(elapsed_ms) + return raw_times + + def _run_mps( + self, + fn: KernelCallable, + args: tuple[Any, ...], + kwargs: dict[str, Any], + n_warmup: int, + n_rounds: int, + do_flush: bool, + ) -> list[float]: + """MPS timing loop using device-level sync + ``time.perf_counter()``. + + SYNTHESIS §3 (load-bearing): we MUST NOT use the CUDA-style pattern + ``start.record(); end.record(); end.synchronize(); start.elapsed_time(end)`` + on MPS — pytorch#162872 deadlocks the calling thread. The + device-level ``torch.mps.synchronize()`` is documented and stable on + PyTorch 2.6+. + """ + import time + + import torch + + if do_flush: + warnings.warn( + "flush_l2=True ignored on MPS (no Apple GPU L2 flush primitive); " + "benchmark stability may be lower than on CUDA", + UserWarning, + stacklevel=2, + ) + + # Warmup + torch.mps.synchronize() + for _ in range(n_warmup): + fn(*args, **kwargs) + torch.mps.synchronize() + + raw_times: list[float] = [] + for _ in range(n_rounds): + torch.mps.synchronize() + t0 = time.perf_counter() + fn(*args, **kwargs) + # Device-level sync — see SYNTHESIS §3 / pytorch#162872. + torch.mps.synchronize() + elapsed_ms = (time.perf_counter() - t0) * 1000.0 + raw_times.append(elapsed_ms) + return raw_times + @pytest.fixture() def gpu_benchmark() -> _BenchmarkRunner: - """Provide a GPU kernel benchmarker using CUDA event timing. + """Provide a GPU kernel benchmarker. + + Uses ``torch.cuda.Event`` timing on NVIDIA, ``torch.mps.synchronize()`` + + wall clock on Apple Silicon (pytorch#162872 deadlock-safe pattern). Usage:: diff --git a/src/gpucheck/fixtures/gpu.py b/src/gpucheck/fixtures/gpu.py index 8717cd7..8e6b839 100644 --- a/src/gpucheck/fixtures/gpu.py +++ b/src/gpucheck/fixtures/gpu.py @@ -2,7 +2,6 @@ from __future__ import annotations -import contextlib import gc import warnings from dataclasses import dataclass @@ -13,6 +12,8 @@ if TYPE_CHECKING: from collections.abc import Generator + from gpucheck.arch.detection import GPUInfo + @dataclass(frozen=True, slots=True) class GPUDevice: @@ -40,85 +41,37 @@ def __str__(self) -> str: ) -def _detect_gpu_pynvml() -> GPUDevice | None: - """Detect GPU using pynvml (no torch dependency).""" - try: - import pynvml - except ImportError: - return None - - try: - pynvml.nvmlInit() - except pynvml.NVMLError: - return None +def _to_device(info: GPUInfo) -> GPUDevice: + """Adapt a richer ``arch.detection.GPUInfo`` into the local ``GPUDevice``. - try: - count = pynvml.nvmlDeviceGetCount() - if count == 0: - return None - - handle = pynvml.nvmlDeviceGetHandleByIndex(0) - name = pynvml.nvmlDeviceGetName(handle) - if isinstance(name, bytes): - name = name.decode("utf-8") - - mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle) - - # Compute capability - major = pynvml.nvmlDeviceGetCudaComputeCapability(handle) - if isinstance(major, tuple): - cc = (major[0], major[1]) - else: - # Older pynvml versions return two separate values - minor = 0 - cc = (major, minor) - - return GPUDevice( - device_id=0, - name=name, - compute_capability=cc, - memory_total=mem_info.total, - memory_free=mem_info.free, - ) - except pynvml.NVMLError: - return None - finally: - with contextlib.suppress(pynvml.NVMLError): - pynvml.nvmlShutdown() + ``GPUInfo`` carries memory in MB; ``GPUDevice`` exposes raw bytes. + The 1MB granularity loss is acceptable for fixture-level reporting + (``__str__`` formats only to whole-MB anyway). + """ + return GPUDevice( + device_id=info.device_id, + name=info.name, + compute_capability=info.compute_capability, + memory_total=info.memory_total_mb * 1024 * 1024, + memory_free=info.memory_free_mb * 1024 * 1024, + ) -def _detect_gpu_torch() -> GPUDevice | None: - """Detect GPU using torch.cuda.""" - try: - import torch - except ImportError: - return None +def detect_gpu() -> GPUDevice | None: + """Auto-detect a GPU, preferring pynvml (lighter) over torch. - if not torch.cuda.is_available(): - return None + Delegates to :func:`gpucheck.arch.detection.detect_gpus` so there is + exactly one detection codepath in the codebase (T-20); ``detect_gpus`` + is ``lru_cache``-backed, so repeated calls are O(1) and the + "no detection backend available" warning fires at most once per session. + The first detected device is adapted to the ``GPUDevice`` shape. + """ + from gpucheck.arch.detection import detect_gpus - try: - device_id = 0 - props = torch.cuda.get_device_properties(device_id) - mem_free, mem_total = torch.cuda.mem_get_info(device_id) - - return GPUDevice( - device_id=device_id, - name=props.name, - compute_capability=(props.major, props.minor), - memory_total=mem_total, - memory_free=mem_free, - ) - except (RuntimeError, AssertionError): + gpus = detect_gpus() + if not gpus: return None - - -def detect_gpu() -> GPUDevice | None: - """Auto-detect a GPU, preferring pynvml (lighter) over torch.""" - device = _detect_gpu_pynvml() - if device is not None: - return device - return _detect_gpu_torch() + return _to_device(gpus[0]) def _cleanup_gpu() -> None: diff --git a/src/gpucheck/fuzzing/__init__.py b/src/gpucheck/fuzzing/__init__.py index ce1098e..a34ee51 100644 --- a/src/gpucheck/fuzzing/__init__.py +++ b/src/gpucheck/fuzzing/__init__.py @@ -7,6 +7,14 @@ from gpucheck.fuzzing.inputs import edge_inputs, mixed_inputs, random_inputs from gpucheck.fuzzing.shapes import ShapeStrategy, fuzz_shapes +from gpucheck.fuzzing.strides import ( + CATEGORIES as STRIDE_CATEGORIES, +) +from gpucheck.fuzzing.strides import ( + StrideStrategy, + fuzz_strides, + fuzz_strides_for_category, +) _LAZY_MAP: dict[str, tuple[str, str]] = { "gpu_shapes": ("gpucheck.fuzzing.strategies", "gpu_shapes"), @@ -30,4 +38,8 @@ def __getattr__(name: str) -> Any: "ShapeStrategy", "gpu_shapes", "gpu_tensors", + "fuzz_strides", + "fuzz_strides_for_category", + "StrideStrategy", + "STRIDE_CATEGORIES", ] diff --git a/src/gpucheck/fuzzing/strides.py b/src/gpucheck/fuzzing/strides.py new file mode 100644 index 0000000..7b95be6 --- /dev/null +++ b/src/gpucheck/fuzzing/strides.py @@ -0,0 +1,367 @@ +"""Stride and contiguity fuzzing for GPU kernels. + +GPU kernel bugs often hide behind non-contiguous tensor layouts: a kernel +might be correct for ``tensor.contiguous()`` but mis-handle a transposed +view, a broadcast-induced stride-0 dim, or a slice with non-unit stride. +This module generates a deterministic corpus of seven stride categories, +plus a Hypothesis :class:`StrideStrategy` for property-based testing. + +Categories (priority order; row_major first as the baseline):: + + row_major -- contiguous, default torch.empty(shape) + column_major -- ATen 'F' layout via transpose-of-contiguous + broadcast -- stride-0 dim (expand) + transpose -- 2D stride permutation + slice -- regular non-unit stride (every-other) + non_contig -- view that is non-contiguous AND not a clean transpose + gather -- irregular access (gather-induced stride pattern) + +The canonical names are snake_case (Python convention). Earlier docs used +kebab-case (``row-major``, ``broadcast-induced``); those forms are +accepted by :func:`fuzz_strides_for_category` and +:func:`fuzz_strides` for backward compatibility, but emit a +:class:`DeprecationWarning` and route through the canonical name. See +:func:`_canonicalize_category` for the alias table. + +Each category is independently chosen because each exercises a different +code path inside PyTorch's kernel dispatcher. A v1.0 test that passes +``row_major`` and fails ``broadcast`` has likely tripped over a missing +broadcast-aware kernel branch. + +The module is **lazy** with respect to torch — it raises +:class:`RuntimeError` on first call if ``torch`` isn't installed, mirroring +the rest of ``gpucheck.fuzzing``. +""" + +from __future__ import annotations + +import warnings +from typing import Any + +CATEGORIES: tuple[str, ...] = ( + "row_major", + "column_major", + "broadcast", + "transpose", + "slice", + "non_contig", + "gather", +) + +# Kebab-case aliases used in earlier MIGRATION.md / CHANGELOG snippets. +# Accepted with a DeprecationWarning so users who copy-pasted the old docs +# don't hit ``ValueError: Unknown stride category 'broadcast-induced'``. +# Maps deprecated → canonical. +_CATEGORY_ALIASES: dict[str, str] = { + "row-major": "row_major", + "column-major": "column_major", + "broadcast-induced": "broadcast", + # ``transpose`` and ``slice`` and ``gather`` are identical in both + # spellings, so they don't need entries here. + "non-contig": "non_contig", + "non-contiguous": "non_contig", + "contiguous-after-clone": "non_contig", + "gather-induced": "gather", +} + + +def _canonicalize_category(category: str) -> str: + """Return the canonical snake_case category name. + + If ``category`` is a known kebab-case alias (review BLOCKER A1), emit + a :class:`DeprecationWarning` and return the canonical mapping. If + it's already canonical, return as-is. Unknown values are returned + unchanged so the caller's ``ValueError`` surfaces with the original + bad name. + """ + if category in _CATEGORY_FN: + return category + canonical = _CATEGORY_ALIASES.get(category) + if canonical is None: + return category + warnings.warn( + f"Stride category {category!r} is deprecated; use " + f"{canonical!r} (snake_case is the canonical form). " + "kebab-case aliases will be removed in a future release.", + DeprecationWarning, + stacklevel=3, + ) + return canonical + + +def _torch_mod() -> Any: + try: + import torch + + return torch + except ImportError as exc: # pragma: no cover -- exercised when torch absent + raise RuntimeError( + "gpucheck.fuzzing.strides requires PyTorch: pip install gpucheck[torch]" + ) from exc + + +def _row_major(shape: tuple[int, ...], dtype: Any, device: str, gen: Any) -> Any: + torch = _torch_mod() + return torch.randn(shape, generator=gen, dtype=torch.float32).to( + dtype=dtype, device=device, + ).contiguous() + + +def _column_major(shape: tuple[int, ...], dtype: Any, device: str, gen: Any) -> Any: + """Column-major: build the contiguous transposed shape, then transpose back. + + For ndim < 2 the concept is undefined; we fall back to row_major. + """ + torch = _torch_mod() + if len(shape) < 2: + return _row_major(shape, dtype, device, gen) + transposed = (shape[1], shape[0]) + shape[2:] + base = torch.randn(transposed, generator=gen, dtype=torch.float32).to( + dtype=dtype, device=device, + ).contiguous() + return base.transpose(0, 1) + + +def _broadcast(shape: tuple[int, ...], dtype: Any, device: str, gen: Any) -> Any: + """Broadcast-induced stride-0 dim along the LAST axis. + + For shape (M, N, K), build a contiguous (M, N, 1) tensor and expand to + (M, N, K). The last dim has stride 0 — kernels that scan strides + naively will multiply-count or read past bounds. + """ + torch = _torch_mod() + if not shape: + return torch.empty(shape, dtype=dtype, device=device) + base_shape = shape[:-1] + (1,) + base = torch.randn(base_shape, generator=gen, dtype=torch.float32).to( + dtype=dtype, device=device, + ).contiguous() + return base.expand(shape) + + +def _transpose(shape: tuple[int, ...], dtype: Any, device: str, gen: Any) -> Any: + """2D-style stride permutation: builds the transposed shape contiguous, + transposes, returns. Different from :func:`_column_major` only in that + transpose dims may be non-(0, 1) for higher-rank tensors — we transpose + the LAST two dims for ndim >= 2. + """ + torch = _torch_mod() + if len(shape) < 2: + return _row_major(shape, dtype, device, gen) + # Transpose last two dims, e.g. (B, M, N) -> build (B, N, M) contiguous, + # then .transpose(-1, -2) to recover (B, M, N) with permuted strides. + transposed = shape[:-2] + (shape[-1], shape[-2]) + base = torch.randn(transposed, generator=gen, dtype=torch.float32).to( + dtype=dtype, device=device, + ).contiguous() + return base.transpose(-1, -2) + + +def _slice(shape: tuple[int, ...], dtype: Any, device: str, gen: Any) -> Any: + """Regular non-unit stride via every-other slicing. + + Build a tensor with each dim doubled, then slice ``[::2, ::2, ...]``. + The resulting view has stride 2 in every dim. + """ + torch = _torch_mod() + if not shape: + return _row_major(shape, dtype, device, gen) + big_shape = tuple(d * 2 for d in shape) + base = torch.randn(big_shape, generator=gen, dtype=torch.float32).to( + dtype=dtype, device=device, + ).contiguous() + slicer = tuple(slice(None, None, 2) for _ in shape) + return base[slicer] + + +def _non_contig(shape: tuple[int, ...], dtype: Any, device: str, gen: Any) -> Any: + """Non-contiguous view that is NOT a clean transpose or slice. + + For ndim >= 3, we permute dims (1, 0, 2, ...). For ndim 2, we transpose + and then slice the last dim by 2 — guaranteed non-contiguous and not a + pure transpose. For ndim 1, fall back to slice. + """ + torch = _torch_mod() + if len(shape) <= 1: + return _slice(shape, dtype, device, gen) + if len(shape) == 2: + big = (shape[1], shape[0] * 2) + base = torch.randn(big, generator=gen, dtype=torch.float32).to( + dtype=dtype, device=device, + ).contiguous() + # transpose then slice the now-first dim by 2 + return base.transpose(0, 1)[::2] + # ndim >= 3 — permute first two dims + transposed = (shape[1], shape[0]) + shape[2:] + base = torch.randn(transposed, generator=gen, dtype=torch.float32).to( + dtype=dtype, device=device, + ).contiguous() + return base.transpose(0, 1) + + +def _gather(shape: tuple[int, ...], dtype: Any, device: str, gen: Any) -> Any: + """Irregular-access tensor: gather a contiguous source by a random index. + + The result is a contiguous tensor of the right shape, but it was + materialized via gather — so kernels that combine gather + reduction + in fused patterns may exhibit different behavior than a pure + contiguous input. (We return the gathered view contiguous; the test + harness's value is in *how* it was built, not the runtime layout.) + """ + torch = _torch_mod() + if not shape: + return _row_major(shape, dtype, device, gen) + src = torch.randn(shape, generator=gen, dtype=torch.float32).to( + dtype=dtype, device=device, + ).contiguous().flatten() + numel = src.numel() + idx = torch.randperm(numel, generator=gen) + return src[idx].reshape(shape).contiguous() + + +_CATEGORY_FN: dict[str, Any] = { + "row_major": _row_major, + "column_major": _column_major, + "broadcast": _broadcast, + "transpose": _transpose, + "slice": _slice, + "non_contig": _non_contig, + "gather": _gather, +} + + +def fuzz_strides_for_category( + shape: tuple[int, ...], + dtype: Any, + category: str, + *, + device: str = "cpu", + seed: int | None = None, +) -> Any: + """Build a single tensor of the given stride category. + + Use this when a test is parametrized over categories (typically via + :func:`parametrize_gpu(stride_categories=...)`). + + Both snake_case (canonical) and kebab-case (deprecated alias) names + are accepted; passing a kebab-case name emits a + :class:`DeprecationWarning`. + """ + category = _canonicalize_category(category) + if category not in _CATEGORY_FN: + raise ValueError( + f"Unknown stride category {category!r}; " + f"expected one of {sorted(CATEGORIES)}" + ) + torch = _torch_mod() + gen: Any = None + if seed is not None: + gen = torch.Generator() + gen.manual_seed(seed) + return _CATEGORY_FN[category](shape, dtype, device, gen) + + +def fuzz_strides( + shape: tuple[int, ...], + dtype: Any, + *, + n: int | None = None, + device: str = "cpu", + seed: int | None = None, + categories: tuple[str, ...] | None = None, +) -> list[tuple[str, Any]]: + """Return a deterministic ``[(category, tensor), ...]`` corpus. + + Parameters + ---------- + shape: + Tensor shape used for every category. + dtype: + torch dtype for every tensor. + n: + Cap on the number of items returned. ``None`` returns all + configured categories (default 7). + device: + Target device string. + seed: + Optional torch RNG seed for reproducibility. + categories: + Override the default category order. Useful for tests that want + only a subset (e.g. only the non-contiguous flavors). + """ + raw_cats = tuple(categories) if categories else CATEGORIES + # Normalize kebab-case aliases (review BLOCKER A1). _canonicalize_category + # emits DeprecationWarning per non-canonical name. + cats = tuple(_canonicalize_category(c) for c in raw_cats) + invalid = [c for c in cats if c not in _CATEGORY_FN] + if invalid: + raise ValueError( + f"Unknown stride categories: {invalid}; " + f"expected from {sorted(CATEGORIES)}" + ) + out: list[tuple[str, Any]] = [] + for cat in cats: + out.append((cat, fuzz_strides_for_category( + shape, dtype, cat, device=device, seed=seed, + ))) + if n is not None: + out = out[:n] + return out + + +# --------------------------------------------------------------------------- +# Hypothesis strategy +# --------------------------------------------------------------------------- + +class StrideStrategy: + """Hypothesis-compatible factory that draws a stride-perturbed tensor. + + Mirrors :class:`gpucheck.fuzzing.ShapeStrategy`'s ``__new__``-as-factory + pattern so callers can write:: + + from hypothesis import given + @given(t=StrideStrategy(shape=(64, 64), dtype=torch.float32)) + def test_kernel_handles_strides(t): ... + + Hypothesis will draw one of the seven categories per test case and + shrink towards ``row_major``. + """ + + def __new__( + cls, + shape: tuple[int, ...], + dtype: Any = None, + *, + device: str = "cpu", + categories: tuple[str, ...] | None = None, + ) -> Any: + try: + from hypothesis import strategies as st + except ImportError as exc: + raise RuntimeError( + "StrideStrategy requires hypothesis: pip install gpucheck[hypothesis]" + ) from exc + + torch = _torch_mod() + if dtype is None: + dtype = torch.float32 + + cats = tuple(categories) if categories else CATEGORIES + + @st.composite + def _draw(draw: Any) -> Any: + cat = draw(st.sampled_from(cats)) + seed = draw(st.integers(min_value=0, max_value=2**31 - 1)) + return fuzz_strides_for_category( + shape, dtype, cat, device=device, seed=seed, + ) + + return _draw() + + +__all__ = [ + "CATEGORIES", + "fuzz_strides", + "fuzz_strides_for_category", + "StrideStrategy", +] diff --git a/src/gpucheck/plugin.py b/src/gpucheck/plugin.py index 86163c6..1de6e2a 100644 --- a/src/gpucheck/plugin.py +++ b/src/gpucheck/plugin.py @@ -2,6 +2,7 @@ from __future__ import annotations +import warnings from typing import Any import pytest @@ -47,6 +48,57 @@ def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line("markers", "gpu: marks tests requiring a GPU") config.addinivalue_line("markers", "slow: marks slow-running tests") config.addinivalue_line("markers", "multi_gpu: marks tests requiring multiple GPUs") + config.addinivalue_line("markers", "mps: marks tests requiring Apple Silicon MPS") + + # Load tolerances + MPS xfail registry from pyproject.toml at session start. + _load_pyproject_config(config.rootpath) + + +def _load_pyproject_config(rootpath: Any) -> None: + """Read ``pyproject.toml`` and apply gpucheck's tool sections. + + Best-effort — if the file is absent gpucheck falls back to its built-in + defaults. If the file exists but is unreadable or malformed, a + :class:`UserWarning` is emitted so the user sees the misconfiguration + rather than silently shipping defaults (security finding PM-2). + + Uses stdlib ``tomllib`` (Python 3.11+) or ``tomli`` (3.10) — both ship with + the python toolchain we target. + """ + from pathlib import Path as _Path + + pyproject = _Path(str(rootpath)) / "pyproject.toml" + if not pyproject.is_file(): + return + + # Python 3.11+ ships tomllib in the stdlib; 3.10 needs `tomli`. + # Both modules import as a name local to this function — mypy's + # static view doesn't know which Python we'll actually run on, so + # the import errors are silenced via the broad mypy override below. + try: + import tomllib + except ModuleNotFoundError: # pragma: no cover -- 3.10 fallback + import tomli as tomllib # type: ignore[no-redef,unused-ignore] + + try: + with pyproject.open("rb") as f: + data = tomllib.load(f) + except (OSError, tomllib.TOMLDecodeError) as exc: + warnings.warn( + f"gpucheck: failed to load {pyproject} ({type(exc).__name__}: {exc}); " + "falling back to built-in tolerance defaults.", + UserWarning, + stacklevel=2, + ) + return + + from gpucheck.assertions.tolerances import ( + apply_config_tolerances, + apply_mps_xfail_config, + ) + + apply_config_tolerances(data) + apply_mps_xfail_config(data) def pytest_collection_modifyitems( diff --git a/src/gpucheck/reporting/__init__.py b/src/gpucheck/reporting/__init__.py index ec31026..6679bef 100644 --- a/src/gpucheck/reporting/__init__.py +++ b/src/gpucheck/reporting/__init__.py @@ -11,6 +11,7 @@ "emit_github_annotations": ("gpucheck.reporting.ci", "emit_github_annotations"), "write_junit_xml": ("gpucheck.reporting.ci", "write_junit_xml"), "generate_pr_comment": ("gpucheck.reporting.ci", "generate_pr_comment"), + "HTMLReporter": ("gpucheck.reporting.html", "HTMLReporter"), } @@ -28,4 +29,5 @@ def __getattr__(name: str) -> Any: "emit_github_annotations", "write_junit_xml", "generate_pr_comment", + "HTMLReporter", ] diff --git a/src/gpucheck/reporting/html.py b/src/gpucheck/reporting/html.py new file mode 100644 index 0000000..e1ec7db --- /dev/null +++ b/src/gpucheck/reporting/html.py @@ -0,0 +1,255 @@ +"""Static HTML dashboard generator for gpucheck JSON run records. + +Reads a ``results.json`` produced by :class:`gpucheck.reporting.json.JSONReporter` +and writes a single self-contained HTML file: zero external CSS, zero +external JavaScript, no fetches at view time. Inline SVG renders the +benchmark bar chart so the file works on a flight without WiFi. + +Sections (in order): + +1. **Summary** — total tests, pass count, fail count, skip count, GPU info. +2. **Test results table** — one row per test with status pill and + collapsed message via ``
``. +3. **Benchmark table** — one row per kernel; inline SVG bar chart of + median timings for at-a-glance regression spotting. +4. **Memory table** — peak / leaked MB per test. +5. **Comparison band** — when a comparison diff is supplied, surfaces + regression / ok / new / removed rows in red / green / blue / gray. + +The renderer is deliberately small (no Jinja, no D3) so it vendorizes +cleanly. Callers needing richer charts can post-process the JSON in any +external dashboard. +""" + +from __future__ import annotations + +import html +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +_STATUS_PILL_BG: dict[str, str] = { + "passed": "#1f7a4d", + "failed": "#a8231f", + "error": "#a8231f", + "skipped": "#a87a1f", + "ok": "#1f7a4d", + "regression": "#a8231f", + "new": "#1f5fa8", + "removed": "#666666", +} + + +@dataclass +class HTMLReporter: + """Render a JSON run record into a self-contained HTML file.""" + + json_path: str | Path + comparison: dict[str, Any] | None = None + title: str = "gpucheck dashboard" + _data: dict[str, Any] = field(default_factory=dict, init=False, repr=False) + + def _load(self) -> dict[str, Any]: + if not self._data: + self._data = json.loads(Path(self.json_path).read_text(encoding="utf-8")) + return self._data + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def render(self, out_path: str | Path) -> Path: + """Write the dashboard to *out_path* and return its :class:`Path`.""" + data = self._load() + out = Path(out_path) + out.parent.mkdir(parents=True, exist_ok=True) + + sections = [ + _render_head(self.title), + _render_summary(data), + _render_test_results(data), + _render_benchmarks(data), + _render_memory(data), + _render_comparison(self.comparison) if self.comparison else "", + _render_foot(), + ] + out.write_text("\n".join(s for s in sections if s) + "\n", encoding="utf-8") + return out + + +# --------------------------------------------------------------------------- +# Section renderers +# --------------------------------------------------------------------------- + + +def _esc(value: Any) -> str: + return html.escape(str(value), quote=True) + + +def _pill(status: str) -> str: + bg = _STATUS_PILL_BG.get(status, "#666666") + return ( + f'' + f"{_esc(status.upper())}" + ) + + +def _render_head(title: str) -> str: + return f""" + +{_esc(title)} + + +

{_esc(title)}

""" + + +def _render_foot() -> str: + return "" + + +def _render_summary(data: dict[str, Any]) -> str: + results = data.get("test_results", []) + passed = sum(1 for r in results if r.get("status") == "passed") + failed = sum(1 for r in results if r.get("status") == "failed") + skipped = sum(1 for r in results if r.get("status") == "skipped") + + gpu_info = data.get("gpu_info", {}) or {} + gpu_summary = " | ".join( + f"{_esc(k)}: {_esc(v)}" for k, v in list(gpu_info.items())[:5] + ) or "no GPU info recorded" + + return f"""

Summary

+
+
{passed}
passed
+
{failed}
failed
+
{skipped}
skipped
+
{gpu_summary}
+
+

timestamp: {_esc(data.get("timestamp", "unknown"))}

""" + + +def _render_test_results(data: dict[str, Any]) -> str: + results = data.get("test_results", []) + if not results: + return "" + rows = [] + for r in results: + status = r.get("status", "unknown") + klass = "regression" if status in {"failed", "error"} else "passed-row" + msg = r.get("message", "") + msg_cell = ( + f'
view
{_esc(msg)}
' if msg else "" + ) + rows.append( + f'{_esc(r.get("name", ""))}' + f'{_pill(status)}' + f'{r.get("duration", 0.0):.4f}s' + f'{msg_cell}', + ) + body = "\n".join(rows) + return f"""

Test Results

+ +{body} +
TestStatusDurationMessage
""" + + +def _render_benchmarks(data: dict[str, Any]) -> str: + benches = data.get("benchmarks", []) + if not benches: + return "" + max_med = max((b.get("median_ms", 0.0) or 0.0) for b in benches) or 1.0 + + rows = [] + for b in benches: + med = float(b.get("median_ms", 0.0) or 0.0) + std = float(b.get("std_ms", 0.0) or 0.0) + bar_w = max(2, int(180 * (med / max_med))) + rows.append( + f'{_esc(b.get("name", ""))}' + f'{med:.3f} ms' + f'{std:.3f} ms' + f'{b.get("samples", 0)}' + f'', + ) + return f"""

Benchmarks

+ + +{"".join(rows)} +
KernelMedianStdSamplesDistribution (relative)
""" + + +def _render_memory(data: dict[str, Any]) -> str: + mem = data.get("memory", []) + if not mem: + return "" + rows = [] + for m in mem: + leaked = float(m.get("leaked_mb", 0.0) or 0.0) + klass = "regression" if leaked > 0 else "passed-row" + rows.append( + f'{_esc(m.get("name", ""))}' + f'{m.get("peak_mb", 0):.2f} MB' + f'{leaked:.2f} MB' + f'{m.get("allocations", 0)}', + ) + return f"""

Memory

+ +{"".join(rows)} +
TestPeakLeakedAllocations
""" + + +def _render_comparison(diff: dict[str, Any]) -> str: + benches = diff.get("benchmarks", []) + if not benches: + return "" + rows = [] + for b in benches: + status = b.get("status", "ok") + klass = status if status in {"regression", "new", "removed"} else "passed-row" + base = b.get("baseline_median_ms", "-") + curr = b.get("current_median_ms", "-") + delta = b.get("delta_pct", 0) + base_str = f"{base:.3f} ms" if isinstance(base, (int, float)) else _esc(base) + curr_str = f"{curr:.3f} ms" if isinstance(curr, (int, float)) else _esc(curr) + rows.append( + f'{_esc(b.get("name", ""))}' + f'{base_str}{curr_str}' + f'{delta:+.1f}%' + f'{_pill(status)}', + ) + return f"""

Comparison vs Baseline

+ + +{"".join(rows)} +
KernelBaselineCurrentDeltaStatus
""" + + +__all__ = ["HTMLReporter"] diff --git a/src/gpucheck/sanitizers/__init__.py b/src/gpucheck/sanitizers/__init__.py index fc5aa12..d1574c4 100644 --- a/src/gpucheck/sanitizers/__init__.py +++ b/src/gpucheck/sanitizers/__init__.py @@ -2,17 +2,31 @@ from __future__ import annotations -from gpucheck.sanitizers.memory import SanitizerMemoryReport, check_memory_leaks, memory_guard +from gpucheck.sanitizers.determinism import ( + DeterminismError, + assert_deterministic, + requires_determinism, +) +from gpucheck.sanitizers.memory import ( + MemoryGuardReport, + SanitizerMemoryReport, + check_memory_leaks, + memory_guard, +) from gpucheck.sanitizers.race import SanitizerReport, run_with_sanitizer # Backward-compat alias MemoryReport = SanitizerMemoryReport __all__ = [ + "MemoryGuardReport", "MemoryReport", "SanitizerMemoryReport", "SanitizerReport", "check_memory_leaks", "memory_guard", "run_with_sanitizer", + "assert_deterministic", + "requires_determinism", + "DeterminismError", ] diff --git a/src/gpucheck/sanitizers/determinism.py b/src/gpucheck/sanitizers/determinism.py new file mode 100644 index 0000000..43b060a --- /dev/null +++ b/src/gpucheck/sanitizers/determinism.py @@ -0,0 +1,190 @@ +"""Determinism sanitizer — assert byte-identical outputs across runs. + +Unlike CUDA, MPS is best-effort deterministic (research SYNTHESIS §4): +PyTorch documentation is silent on Apple Silicon determinism guarantees, +and the empirical record (pytorch#181936, #170837, #177116) shows real +run-to-run divergence. This module provides: + +- :func:`assert_deterministic` — runs ``fn`` ``n`` times under fixed seeds + and asserts every output tensor is byte-identical to the first run. + On MPS, structured failure surfaces ``DeterminismError`` so callers + know whether the divergence is at the precision floor (acceptable in + some pipelines) or a literal inconsistency. +- :func:`requires_determinism` — function decorator: wraps the test body + so that calling it n times is the test (instead of the test author + having to write the loop themselves). + +The seeded run sets: + + torch.manual_seed(seed) + if mps available: torch.mps.manual_seed(seed) + if cuda available: torch.cuda.manual_seed_all(seed) + random.seed(seed); numpy.random.seed(seed) +""" + +from __future__ import annotations + +import functools +import random +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Callable + + +class DeterminismError(AssertionError): + """Raised when ``assert_deterministic`` observes diverging outputs.""" + + +def _seed_all(seed: int) -> None: + """Seed every RNG we know about. Best-effort — silent on missing modules.""" + random.seed(seed) + try: + import numpy as np + + np.random.seed(seed) + except ImportError: + pass + try: + import torch + + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + mps = getattr(torch.backends, "mps", None) + if mps is not None and mps.is_available(): + seed_fn = getattr(torch.mps, "manual_seed", None) + if callable(seed_fn): + seed_fn(seed) + except ImportError: + pass + + +def _equal(a: Any, b: Any, *, atol: float = 0.0, rtol: float = 0.0) -> bool: + """Compare two outputs. + + Default mode (``atol=rtol=0``) is byte-identical equality via + :func:`torch.equal`. When either tolerance is non-zero, falls back + to :func:`torch.allclose` (the right contract for MPS, where + research SYNTHESIS §4 documents best-effort determinism). + + Handles torch.Tensor (same device, same dtype), tuples / lists + elementwise, and falls back to ``==`` for non-tensor scalars. + """ + try: + import torch + + if isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor): + if a.shape != b.shape or a.dtype != b.dtype or a.device != b.device: + return False + if atol == 0.0 and rtol == 0.0: + return bool(torch.equal(a, b)) + return bool(torch.allclose(a, b, atol=atol, rtol=rtol)) + except ImportError: + pass + if isinstance(a, (tuple, list)) and isinstance(b, (tuple, list)): + if len(a) != len(b): + return False + return all( + _equal(x, y, atol=atol, rtol=rtol) + for x, y in zip(a, b, strict=False) + ) + return bool(a == b) + + +def assert_deterministic( + fn: Callable[..., Any], + *args: Any, + n: int = 3, + seed: int = 0, + atol: float = 0.0, + rtol: float = 0.0, + **kwargs: Any, +) -> Any: + """Run *fn* ``n`` times under fixed seeds; assert outputs match. + + Parameters + ---------- + fn: + Callable producing the output to compare. May return a tensor, a + tuple of tensors, or any equality-comparable value. + *args / **kwargs: + Forwarded to *fn*. + n: + Number of repetitions. Must be ``>= 2``. + seed: + Seed applied to ``random``, ``numpy.random``, ``torch.manual_seed``, + ``torch.cuda.manual_seed_all``, and ``torch.mps.manual_seed`` (if + available) before each call. + atol, rtol: + Tolerance knobs for the cross-run comparison. Default ``0.0`` + means byte-identical equality (``torch.equal``). When either is + non-zero, comparison switches to ``torch.allclose`` — + appropriate for MPS where research SYNTHESIS §4 documents + best-effort (not bit-exact) determinism. + + Returns + ------- + The output of the first run, so callers can pass through values + that they want to use after asserting determinism. + + Raises + ------ + DeterminismError: + If any run's output differs from the first run beyond the + configured tolerance. + """ + if n < 2: + raise ValueError(f"assert_deterministic requires n >= 2, got {n}") + + _seed_all(seed) + first = fn(*args, **kwargs) + for i in range(1, n): + _seed_all(seed) + candidate = fn(*args, **kwargs) + if not _equal(first, candidate, atol=atol, rtol=rtol): + mode = "byte-identical" if atol == 0.0 and rtol == 0.0 else ( + f"allclose(atol={atol}, rtol={rtol})" + ) + raise DeterminismError( + f"assert_deterministic: run {i} produced output that differs " + f"from run 0 (n={n}, seed={seed}, mode={mode}). On MPS this " + f"can happen legitimately (best-effort determinism per " + f"SYNTHESIS §4); pass `atol=`/`rtol=` to opt into " + f"tolerance-based determinism, widen tolerances via " + f"tolerance_context, or add the op to the " + f"[tool.gpucheck.mps.xfail] block." + ) + return first + + +def requires_determinism( + *, + n: int = 3, + seed: int = 0, + atol: float = 0.0, + rtol: float = 0.0, +) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + """Decorator: invoke the test function ``n`` times under fixed seeds. + + Equivalent to wrapping the test body in + :func:`assert_deterministic`. ``atol``/``rtol`` are forwarded so + MPS users can opt into tolerance-based determinism instead of the + byte-equality default:: + + @requires_determinism(n=5, seed=42, atol=1e-5) + def test_my_kernel(): + x = torch.randn(64, 64, device="mps") + return my_kernel(x) + """ + def decorator(fn: Callable[..., Any]) -> Callable[..., Any]: + @functools.wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + return assert_deterministic( + fn, *args, n=n, seed=seed, atol=atol, rtol=rtol, **kwargs, + ) + return wrapper + return decorator + + +__all__ = ["assert_deterministic", "requires_determinism", "DeterminismError"] diff --git a/src/gpucheck/sanitizers/memory.py b/src/gpucheck/sanitizers/memory.py index 44de7c8..723a9ef 100644 --- a/src/gpucheck/sanitizers/memory.py +++ b/src/gpucheck/sanitizers/memory.py @@ -138,9 +138,46 @@ def check_memory_leaks( ) +@dataclass(slots=True) +class MemoryGuardReport: + """Public report yielded by :func:`memory_guard`. + + Populated after the ``with`` block exits. Distinct from + :class:`gpucheck.fixtures.profiler.MemoryReport` (which is a *frozen* + fixture-side summary) because the guard pattern requires a placeholder + that the context manager can fill on exit. + """ + + leaked_bytes: int = 0 + peak_bytes: int = 0 + allocations: int = 0 + deallocations: int = 0 + + @property + def leaked_mb(self) -> float: + return self.leaked_bytes / (1024 * 1024) + + @property + def peak_mb(self) -> float: + return self.peak_bytes / (1024 * 1024) + + @property + def has_leak(self) -> bool: + return self.leaked_bytes > 0 + + def to_report(self) -> SanitizerMemoryReport: + """Return a frozen :class:`SanitizerMemoryReport` snapshot.""" + return SanitizerMemoryReport( + leaked_bytes=self.leaked_bytes, + peak_bytes=self.peak_bytes, + allocations=self.allocations, + deallocations=self.deallocations, + ) + + @contextmanager -def memory_guard(threshold_bytes: int = 0) -> Generator[_MutableReport, None, None]: - """Context manager that tracks GPU memory and yields a :class:`SanitizerMemoryReport`. +def memory_guard(threshold_bytes: int = 0) -> Generator[MemoryGuardReport, None, None]: + """Context manager that tracks GPU memory and yields a :class:`MemoryGuardReport`. Usage:: @@ -161,7 +198,8 @@ def memory_guard(threshold_bytes: int = 0) -> Generator[_MutableReport, None, No except ImportError: pass - # Use a mutable wrapper so the caller can inspect the report after the block. + # Track the entry-side state in a local dict so the report can be filled + # in after the user's ``with`` block runs. _holder: dict[str, Any] = {} if torch_available: @@ -175,9 +213,8 @@ def memory_guard(threshold_bytes: int = 0) -> Generator[_MutableReport, None, No else: _holder["before"] = _get_pynvml_memory() - # Yield a _MutableReport so caller can inspect after block - mut = _MutableReport() - yield mut + report = MemoryGuardReport() + yield report _sync_and_gc() @@ -191,68 +228,19 @@ def memory_guard(threshold_bytes: int = 0) -> Generator[_MutableReport, None, No alloc_after = stats_after.get("allocation.all.current", 0) free_count = stats_after.get("free.all.current", 0) - mut._fill( - leaked_bytes=max(0, after - _holder["before"]), - peak_bytes=peak, - allocations=max(0, alloc_after - alloc_before), - deallocations=free_count, - ) + report.leaked_bytes = max(0, after - _holder["before"]) + report.peak_bytes = peak + report.allocations = max(0, alloc_after - alloc_before) + report.deallocations = free_count else: after = _get_pynvml_memory() - mut._fill( - leaked_bytes=max(0, after - _holder["before"]), - peak_bytes=max(_holder["before"], after), - allocations=0, - deallocations=0, - ) + report.leaked_bytes = max(0, after - _holder["before"]) + report.peak_bytes = max(_holder["before"], after) + report.allocations = 0 + report.deallocations = 0 - if threshold_bytes > 0 and mut.leaked_bytes > threshold_bytes: + if threshold_bytes > 0 and report.leaked_bytes > threshold_bytes: raise RuntimeError( - f"GPU memory leak detected: {mut.leaked_bytes} bytes " + f"GPU memory leak detected: {report.leaked_bytes} bytes " f"(threshold: {threshold_bytes})" ) - - -class _MutableReport: - """Mutable stand-in for :class:`SanitizerMemoryReport`, filled after context exit.""" - - __slots__ = ("leaked_bytes", "peak_bytes", "allocations", "deallocations") - - def __init__(self) -> None: - self.leaked_bytes: int = 0 - self.peak_bytes: int = 0 - self.allocations: int = 0 - self.deallocations: int = 0 - - def _fill( - self, - *, - leaked_bytes: int, - peak_bytes: int, - allocations: int, - deallocations: int, - ) -> None: - self.leaked_bytes = leaked_bytes - self.peak_bytes = peak_bytes - self.allocations = allocations - self.deallocations = deallocations - - @property - def leaked_mb(self) -> float: - return self.leaked_bytes / (1024 * 1024) - - @property - def peak_mb(self) -> float: - return self.peak_bytes / (1024 * 1024) - - @property - def has_leak(self) -> bool: - return self.leaked_bytes > 0 - - def to_report(self) -> SanitizerMemoryReport: - return SanitizerMemoryReport( - leaked_bytes=self.leaked_bytes, - peak_bytes=self.peak_bytes, - allocations=self.allocations, - deallocations=self.deallocations, - ) diff --git a/src/gpucheck/sanitizers/race.py b/src/gpucheck/sanitizers/race.py index aec6b59..7f482cc 100644 --- a/src/gpucheck/sanitizers/race.py +++ b/src/gpucheck/sanitizers/race.py @@ -8,6 +8,7 @@ import subprocess import sys import tempfile +import warnings from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal @@ -17,6 +18,17 @@ SanitizerTool = Literal["memcheck", "racecheck", "initcheck", "synccheck"] _VALID_TOOLS: frozenset[str] = frozenset({"memcheck", "racecheck", "initcheck", "synccheck"}) +# Allowlist of canonical CUDA install prefixes. ``CUDA_HOME`` / ``CUDA_PATH`` +# values are normalized via ``os.path.realpath`` and rejected if they +# resolve outside this set. Mitigates security finding TM-E1: an attacker +# who can set the env var should not be able to redirect gpucheck into +# executing an arbitrary binary named ``compute-sanitizer``. +_CUDA_HOME_ALLOWLIST: tuple[str, ...] = ( + "/usr/local/cuda", + "/opt/nvidia/cuda", + "/opt/cuda", +) + @dataclass(frozen=True, slots=True) class SanitizerError: @@ -48,20 +60,55 @@ def error_count(self) -> int: def _find_compute_sanitizer() -> str | None: - """Locate compute-sanitizer binary on PATH or in CUDA_HOME.""" + """Locate compute-sanitizer binary on PATH or in CUDA_HOME. + + ``CUDA_HOME`` / ``CUDA_PATH`` env vars are normalized via + ``os.path.realpath`` and validated against + :data:`_CUDA_HOME_ALLOWLIST` before being trusted. Mitigates security + finding TM-E1. + """ path = shutil.which("compute-sanitizer") if path: return path cuda_home = os.environ.get("CUDA_HOME") or os.environ.get("CUDA_PATH", "") - if cuda_home: - candidate = os.path.join(cuda_home, "bin", "compute-sanitizer") - if os.path.isfile(candidate) and os.access(candidate, os.X_OK): - return candidate + if not cuda_home: + return None + + # Resolve symlinks so an attacker can't bypass the allowlist by + # planting a symlink that points outside the trusted prefixes. + real = os.path.realpath(cuda_home) + if not _is_allowed_cuda_home(real): + warnings.warn( + f"CUDA_HOME / CUDA_PATH={cuda_home!r} resolves to {real!r} which is " + f"outside the allowlist {_CUDA_HOME_ALLOWLIST!r}; ignoring " + f"(set CUDA_HOME to a path under one of those prefixes, or " + f"install compute-sanitizer onto PATH).", + RuntimeWarning, + stacklevel=2, + ) + return None + + candidate = os.path.join(real, "bin", "compute-sanitizer") + if os.path.isfile(candidate) and os.access(candidate, os.X_OK): + return candidate return None +def _is_allowed_cuda_home(real_path: str) -> bool: + """Return ``True`` if *real_path* is inside one of the allowlist prefixes. + + The check is exact-prefix-with-separator so ``/usr/local/cuda-evil`` + does NOT match ``/usr/local/cuda``. + """ + norm = os.path.normpath(real_path) + for prefix in _CUDA_HOME_ALLOWLIST: + if norm == prefix or norm.startswith(prefix + os.sep): + return True + return False + + def _parse_sanitizer_output( raw: str, tool: SanitizerTool, ) -> tuple[list[SanitizerError], list[str]]: diff --git a/tests/gpu_integration/conftest.py b/tests/gpu_integration/conftest.py index 6008ebd..ed7438d 100644 --- a/tests/gpu_integration/conftest.py +++ b/tests/gpu_integration/conftest.py @@ -1,4 +1,10 @@ -"""Shared fixtures for GPU integration tests.""" +"""Shared fixtures and skip-gating for GPU integration tests. + +These tests are designed to run on real GPU hardware (CUDA or, opt-in, MPS). +On a host with neither, the entire suite is skipped at collection time so the +README claim ``pytest tests/gpu_integration/`` auto-skips without GPU stays +honest. See docs-tester finding R-B21 / T-B5. +""" from __future__ import annotations @@ -7,6 +13,70 @@ import pytest +def pytest_addoption(parser: pytest.Parser) -> None: + """Add ``--mps-integration`` to opt into running gpu_integration on MPS.""" + parser.addoption( + "--mps-integration", + action="store_true", + default=False, + help=( + "Opt in to running tests/gpu_integration/ on Apple Silicon MPS. " + "Without this flag the suite is skipped on non-CUDA hosts." + ), + ) + + +def _cuda_available() -> bool: + """Return True iff torch reports a usable CUDA device.""" + try: + import torch + except ImportError: + return False + try: + return bool(torch.cuda.is_available()) + except (RuntimeError, AssertionError): + return False + + +def _mps_available() -> bool: + """Return True iff torch reports a usable MPS device.""" + try: + import torch + except ImportError: + return False + try: + return bool(getattr(torch.backends, "mps", None) and torch.backends.mps.is_available()) + except (RuntimeError, AssertionError): + return False + + +def pytest_collection_modifyitems( + config: pytest.Config, items: list[pytest.Item] +) -> None: + """Skip the gpu_integration suite when no usable GPU mode is detected. + + The README claims ``pytest tests/gpu_integration/`` auto-skips without + a GPU. Previously, on MPS hosts those tests collected and then failed + on CUDA-specific calls. We now skip at collection time unless either: + + - CUDA is available (preferred path), or + - MPS is available *and* the user passed ``--mps-integration``. + """ + if _cuda_available(): + return + if config.getoption("--mps-integration") and _mps_available(): + return + + if _mps_available(): + reason = "MPS detected but --mps-integration not set; pass it to opt in" + else: + reason = "no CUDA GPU available (and no --mps-integration flag set)" + + skip_marker = pytest.mark.skip(reason=reason) + for item in items: + item.add_marker(skip_marker) + + @pytest.fixture() def results() -> dict[str, Any]: """Mutable dict for benchmark tests to store their results.""" diff --git a/tests/test_assert_close_contiguous.py b/tests/test_assert_close_contiguous.py new file mode 100644 index 0000000..8482592 --- /dev/null +++ b/tests/test_assert_close_contiguous.py @@ -0,0 +1,89 @@ +"""Tests that `_to_numpy` handles non-contiguous torch tensors. + +Stride-fuzzed / sliced / transposed tensors break ``.numpy()`` on torch <2.1 +with a ``RuntimeError`` ("input array is not C-contiguous"). T-02 adds +``.contiguous()`` to the slow path; these tests pin the fix. + +Source: security-postmerge PM-4; planner T-02. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np +import pytest + +if TYPE_CHECKING: + from collections.abc import Callable + +torch = pytest.importorskip("torch") + +from gpucheck.assertions.close import _to_numpy, assert_close # noqa: E402 + + +# Three canonical non-contiguous stride patterns. +def _slice_pattern() -> torch.Tensor: + """Sliced row of a 2D tensor — non-contiguous when row stride != col.""" + base = torch.arange(64, dtype=torch.float32).reshape(8, 8) + sliced = base[:, ::2] # stride 2 columns: not C-contiguous + assert not sliced.is_contiguous() + return sliced + + +def _transpose_pattern() -> torch.Tensor: + """Transposed 2D tensor — non-contiguous (row/col strides swapped).""" + base = torch.arange(64, dtype=torch.float32).reshape(8, 8) + t = base.t() + assert not t.is_contiguous() + return t + + +def _broadcast_pattern() -> torch.Tensor: + """Broadcast view via ``.expand`` — has zero strides on broadcast dims.""" + base = torch.arange(8, dtype=torch.float32).reshape(8, 1) + expanded = base.expand(8, 4) + assert not expanded.is_contiguous() + return expanded + + +@pytest.mark.parametrize( + ("name", "factory"), + [ + ("slice", _slice_pattern), + ("transpose", _transpose_pattern), + ("broadcast", _broadcast_pattern), + ], +) +def test_to_numpy_handles_non_contiguous_input( + name: str, factory: Callable[[], torch.Tensor] +) -> None: + """`_to_numpy` must not raise on stride-fuzzed inputs.""" + tensor = factory() + arr = _to_numpy(tensor) + assert isinstance(arr, np.ndarray) + # Shape preserved through the conversion. + assert arr.shape == tuple(tensor.shape), ( + f"{name}: shape mismatch — got {arr.shape}, expected {tuple(tensor.shape)}" + ) + # Values preserved through the conversion. + np.testing.assert_array_equal(arr, tensor.contiguous().numpy()) + + +@pytest.mark.parametrize( + ("name", "factory"), + [ + ("slice", _slice_pattern), + ("transpose", _transpose_pattern), + ("broadcast", _broadcast_pattern), + ], +) +def test_assert_close_handles_non_contiguous_input( + name: str, factory: Callable[[], torch.Tensor] +) -> None: + """End-to-end: `assert_close` should not raise RuntimeError when comparing + non-contiguous tensors against their contiguous equivalents.""" + a = factory() + b = a.contiguous().clone() + # No RuntimeError: comparison flows through `_to_numpy`'s slow path. + assert_close(a, b) diff --git a/tests/test_assert_close_mps.py b/tests/test_assert_close_mps.py new file mode 100644 index 0000000..c3dd59d --- /dev/null +++ b/tests/test_assert_close_mps.py @@ -0,0 +1,92 @@ +"""assert_close on MPS: fast-path widening + 2x tolerance overlay (Track A).""" + +from __future__ import annotations + +import pytest + +from gpucheck.assertions import assert_close, compute_tolerance + + +def _has_mps() -> bool: + try: + import torch + except ImportError: + return False + mps = getattr(torch.backends, "mps", None) + return bool(mps is not None and mps.is_available()) + + +def test_compute_tolerance_mps_doubles_float32() -> None: + base_atol, base_rtol = compute_tolerance("float32") + mps_atol, mps_rtol = compute_tolerance("float32", device_type="mps") + assert mps_atol == pytest.approx(base_atol * 2.0) + assert mps_rtol == pytest.approx(base_rtol * 2.0) + + +def test_compute_tolerance_mps_doubles_float16() -> None: + base_atol, _base_rtol = compute_tolerance("float16") + mps_atol, _mps_rtol = compute_tolerance("float16", device_type="mps") + assert mps_atol == pytest.approx(base_atol * 2.0) + + +def test_compute_tolerance_mps_doubles_bfloat16() -> None: + base_atol, _base_rtol = compute_tolerance("bfloat16") + mps_atol, _mps_rtol = compute_tolerance("bfloat16", device_type="mps") + assert mps_atol == pytest.approx(base_atol * 2.0) + + +def test_compute_tolerance_mps_keeps_float64_unchanged() -> None: + """float64 is rarely load-bearing on MPS; we don't inflate.""" + base = compute_tolerance("float64") + mps = compute_tolerance("float64", device_type="mps") + assert base == mps + + +def test_compute_tolerance_cuda_unchanged_when_device_type_cuda() -> None: + base = compute_tolerance("float32") + cuda = compute_tolerance("float32", device_type="cuda") + assert base == cuda + + +def test_compute_tolerance_with_kdim_and_mps_overlay() -> None: + """MPS overlay applies AFTER k_dim sqrt scaling so the order is documented.""" + cuda = compute_tolerance("float32", k_dim=512) + mps = compute_tolerance("float32", k_dim=512, device_type="mps") + assert mps[0] == pytest.approx(cuda[0] * 2.0) + + +@pytest.mark.skipif(not _has_mps(), reason="MPS not available") +def test_assert_close_mps_fast_path_no_cpu_transfer() -> None: + """On equal MPS tensors, assert_close returns without going through numpy. + + We patch _to_numpy to raise — if the fast-path is taken, _to_numpy is + never called and the test passes; if the slow path is taken, it raises. + """ + import torch + + from gpucheck.assertions import close as close_mod + + original = close_mod._to_numpy + + def trip_wire(*_a, **_kw): + raise AssertionError("_to_numpy was called — fast path missed!") + + close_mod._to_numpy = trip_wire # type: ignore[assignment] + try: + a = torch.ones(8, 8, device="mps") + b = torch.ones(8, 8, device="mps") + assert_close(a, b) + finally: + close_mod._to_numpy = original # type: ignore[assignment] + + +@pytest.mark.skipif(not _has_mps(), reason="MPS not available") +def test_assert_close_mps_passes_with_mps_overlay_for_float16() -> None: + """Two MPS fp16 tensors that differ by ~1.5e-2 must pass under MPS 2x + overlay (base atol=1e-2, MPS atol=2e-2). + """ + import torch + + a = torch.full((16, 16), 1.0, device="mps", dtype=torch.float16) + b = torch.full((16, 16), 1.0 + 1.5e-2, device="mps", dtype=torch.float16) + assert_close(a, b) diff --git a/tests/test_assertions.py b/tests/test_assertions.py index f6acc34..f9b7f55 100644 --- a/tests/test_assertions.py +++ b/tests/test_assertions.py @@ -224,6 +224,117 @@ def test_report_with_nan_inf(self) -> None: assert "Inf" in report +# --------------------------------------------------------------------------- +# format_mismatch_report — pinned numeric fields (T-10, kills ~30 mutants) +# --------------------------------------------------------------------------- + + +class TestMismatchReportPinnedNumerics: + """Lock down the exact numeric values reported by ``format_mismatch_report``. + + These tests target ``assertions/reporting.py`` mutation survivors + documented in `EVIDENCE/mutator-survivors.md` top-leverage #1. Each test + fixes hard-coded expected values so that any arithmetic substitution + (e.g. ``+`` → ``-``, ``np.nanmax`` → ``np.nanmin``, ``unravel_index`` + swap, ``mismatch_pct`` factor flip) breaks the assertion. + """ + + def test_max_abs_error_value_is_pinned(self) -> None: + """Max absolute error == max of element-wise |actual - expected|. + + Construct ``diff = [0.5, 4.5, 1.5, 3.5]`` so the unique maximum is + 4.5 at flat index 1. The report formats with ``{:.6e}``, yielding + ``"4.500000e+00"``. Any swap of ``np.nanmax`` → ``np.nanmin`` / + ``np.nanmean`` / sign flip in the diff computation breaks this. + """ + actual = np.array([1.0, 0.5, 2.5, 0.5], dtype=np.float64) + expected = np.array([1.5, 5.0, 1.0, 4.0], dtype=np.float64) + # Element-wise |a - b| = [0.5, 4.5, 1.5, 3.5] — exact, no FP rounding. + + report = format_mismatch_report(actual, expected, atol=0.0, rtol=0.0) + + # Pinned: the unique maximum 4.5 is rendered as "4.500000e+00". + assert "4.500000e+00" in report, ( + "Max absolute error value drifted from 4.5; " + "check `np.nanmax(diff)` and the `{:.6e}` formatter." + ) + # Mean abs error is (0.5+4.5+1.5+3.5)/4 = 2.5 → "2.500000e+00". + assert "2.500000e+00" in report, ( + "Mean absolute error value drifted from 2.5; " + "check `np.nanmean(diff)`." + ) + # And the table label must be present (kills label-mutation survivors). + assert "Max absolute error" in report + assert "Mean absolute error" in report + + def test_mismatch_count_and_location_are_pinned(self) -> None: + """Mismatch count, percentage, and 2-D max-error location are pinned. + + With ``atol=0, rtol=0``, every element above zero diff is a + mismatch. The 2-D layout pins the unravel_index call: maximum is + at row 1, col 2. + """ + actual = np.zeros((2, 3), dtype=np.float64) + # Place the unique maximum (5.0) at row=1, col=2. + expected = np.array( + [[1.0, 2.0, 3.0], + [4.0, 0.0, 5.0]], + dtype=np.float64, + ) + # Mismatches at 5 of 6 positions (the (1, 1) zero matches). + + report = format_mismatch_report(actual, expected, atol=0.0, rtol=0.0) + + # 5 mismatches out of 6 total → "5 / 6 (83.33%)". + assert "5 / 6 (83.33%)" in report, ( + "Mismatch count / total / pct drifted; check " + "`mismatch_count = int(np.sum(mismatch_mask))` and the pct factor (100.0)." + ) + assert "Mismatch count" in report + # Location: max diff |0 - 5| = 5 sits at (1, 2). Off-by-one or axis + # swap in `np.unravel_index` would yield e.g. "(2, 1)" or "(0, 2)". + assert "(1, 2)" in report, ( + "Location of max error is no longer (1, 2); check " + "`np.unravel_index(np.nanargmax(diff), diff.shape)`." + ) + assert "Location of max error" in report + # And the max abs error itself is 5.0, formatted as "5.000000e+00". + assert "5.000000e+00" in report + + def test_histogram_present_with_pinned_bucket_and_count(self) -> None: + """The error histogram appears with a known bucket label and count. + + Construct 3 mismatches all with absolute error == 1e-3. log10(1e-3) + is -3, so the only bucket is ``[1e-3, 1e-2)`` with count 3. + """ + actual = np.zeros(3, dtype=np.float64) + expected = np.full(3, 1e-3, dtype=np.float64) + + report = format_mismatch_report(actual, expected, atol=0.0, rtol=0.0) + + # Histogram panel header — kills any rename of the panel title. + assert "Error Histogram" in report + # Bucket label format `[1e{lo:+d}, 1e{hi:+d})`. Mutating `lo` or `hi` + # by ±1 changes the rendered label. + assert "[1e-3, 1e-2)" in report, ( + "Histogram bucket label drifted; check `np.floor(np.min(log_vals))` " + "and `np.ceil(np.max(log_vals))`, plus the `{:+d}` format." + ) + # All 3 mismatches fall into the single bucket. The histogram line + # formats as `[1e-3, 1e-2) | ███...███ 3`. Strip ANSI escape codes + # (Rich emits `\x1b[m`) and isolate the bucket line. + import re + plain_report = re.sub(r"\x1b\[[0-9;]*m", "", report) + tail_after_bucket = plain_report.split("[1e-3, 1e-2)")[1] + bucket_line_tail = tail_after_bucket.split("\n", 1)[0] + digits_only = "".join(ch for ch in bucket_line_tail if ch.isdigit()) + assert digits_only == "3", ( + "Histogram count for the [1e-3, 1e-2) bucket is no longer 3 " + f"(got digits={digits_only!r}); " + "check `np.histogram(log_vals, bins=bins)` and the bar-rendering loop." + ) + + # --------------------------------------------------------------------------- # baseline_2x tolerance doubling # --------------------------------------------------------------------------- @@ -253,6 +364,64 @@ def test_baseline_2x_fails_beyond_doubled_tolerance(self) -> None: assert_close(a, b, baseline_2x=True) +class TestBaseline2xKDimScaling: + """Regression test for BLOCKER N1. + + The ``baseline_2x`` path used to scale ``atol`` by ``sqrt(k_dim)`` + while every other code path scales by ``sqrt(k_dim/128)``. At + k_dim=4096 the two diverge by ``sqrt(128) ≈ 11.3×`` — silently + making ``baseline_2x=True`` 11× more permissive than canonical. + The expectation: ``baseline_2x`` is exactly 2× the canonical + tolerance for the same dtype + k_dim. + """ + + @pytest.mark.parametrize("k_dim", [128, 1024, 4096]) + def test_baseline_2x_is_exactly_2x_canonical(self, k_dim: int) -> None: + # Canonical: dtype-aware + sqrt(k_dim/128) scaling. + canonical_atol, canonical_rtol = compute_tolerance("float16", k_dim=k_dim) + + # Build two arrays with diff that sits just inside 2× canonical + # but outside 1× canonical. baseline_2x should pass; canonical + # should fail. + # We deliberately craft the diff at 1.5× canonical_atol so it + # demonstrably fails canonical and passes 2x — and then we + # also pin the boundary at 2.5× to demonstrate it fails the 2x + # threshold (proving 2x is not the bug-prone 256x). + diff = canonical_atol * 1.5 + a = np.array([0.0], dtype=np.float16) + b = np.array([diff], dtype=np.float16) + + # Sanity: canonical (no baseline_2x) rejects diff > 1× canonical. + with pytest.raises(AssertionError): + assert_close(a, b, k_dim=k_dim) + + # baseline_2x must accept diff < 2× canonical at this k_dim. + assert_close(a, b, k_dim=k_dim, baseline_2x=True) + + # baseline_2x must still reject diff > 2× canonical: pin at 2.5×. + a2 = np.array([0.0], dtype=np.float16) + b2 = np.array([canonical_atol * 2.5], dtype=np.float16) + with pytest.raises(AssertionError): + assert_close(a2, b2, k_dim=k_dim, baseline_2x=True) + # rtol unused in this craft (b2 large; expected==0 → rtol leg vanishes). + assert canonical_rtol >= 0.0 + + @pytest.mark.parametrize("k_dim", [128, 1024, 4096]) + def test_baseline_2x_does_not_use_sqrt_k(self, k_dim: int) -> None: + """At k_dim=4096 the buggy sqrt(K) path was ``11.3× looser`` than + the canonical sqrt(K/128) path. Pin the contract so the bug + cannot regress: a diff at ``2.5× canonical_atol`` MUST fail. + Under the old bug, the threshold would be ``2× sqrt(128) ≈ + 22.6×`` of canonical — and 2.5× canonical would erroneously + pass. + """ + canonical_atol, _ = compute_tolerance("float16", k_dim=k_dim) + a = np.array([0.0], dtype=np.float16) + b = np.array([canonical_atol * 2.5], dtype=np.float16) + with pytest.raises(AssertionError): + assert_close(a, b, k_dim=k_dim, baseline_2x=True) + + # --------------------------------------------------------------------------- # Mixed-precision _resolve_dtype # --------------------------------------------------------------------------- diff --git a/tests/test_backends.py b/tests/test_backends.py new file mode 100644 index 0000000..fc537c1 --- /dev/null +++ b/tests/test_backends.py @@ -0,0 +1,158 @@ +"""Backend Protocol contract tests (Track A).""" + +from __future__ import annotations + +import warnings + +import pytest + +from gpucheck.backends import Backend, available_backends, get_backend + + +def _torch(): + try: + import torch + + return torch + except ImportError: + pytest.skip("torch not installed") + + +def test_get_backend_rejects_unknown_name() -> None: + with pytest.raises(ValueError, match="Unknown backend"): + get_backend("rocm") + + +def test_available_backends_returns_list() -> None: + backends = available_backends() + assert isinstance(backends, list) + for b in backends: + assert isinstance(b, Backend) + assert b.name in {"cuda", "mps"} + + +def test_available_backends_priority_cuda_before_mps() -> None: + backends = available_backends() + names = [b.name for b in backends] + if "cuda" in names and "mps" in names: + assert names.index("cuda") < names.index("mps") + + +# --------------------------------------------------------------------------- +# MPS backend (only runs on machines where MPS is available) +# --------------------------------------------------------------------------- + +@pytest.fixture() +def mps_backend(): + torch = _torch() + mps = getattr(torch.backends, "mps", None) + if mps is None or not mps.is_available(): + pytest.skip("MPS not available on this machine") + from gpucheck.backends.mps import MPSBackend + + return MPSBackend() + + +def test_mps_backend_name(mps_backend) -> None: + assert mps_backend.name == "mps" + + +def test_mps_backend_synchronize_no_event_synchronize(mps_backend) -> None: + """SYNTHESIS §3 — MUST use device-level torch.mps.synchronize, NOT + per-event Event.synchronize (deadlocks on Apple Silicon, pytorch#162872). + + We assert this structurally by calling synchronize() and verifying it + returns without raising. Detailed source-introspection lives in the + next test; this one is the smoke check. + """ + mps_backend.synchronize() # Must not hang or raise + + +def test_mps_backend_event_timer_uses_device_sync_not_event_sync(mps_backend) -> None: + """The event_timer context manager must use torch.mps.synchronize(). + + We verify this by source-introspecting the implementation, scanning the + AST so docstring text doesn't trigger false positives. The deadlock + pattern (pytorch#162872) is calling ``.synchronize()`` on a + ``torch.mps.event.Event`` instance. + """ + import ast + import inspect + import textwrap + + from gpucheck.backends.mps import MPSBackend + + src = textwrap.dedent(inspect.getsource(MPSBackend.event_timer)) + tree = ast.parse(src) + + # Find every Call expression in code (not docstrings). + found_device_sync = False + forbidden_calls: list[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute): + attr = node.func + # torch.mps.synchronize() — required pattern. + if ( + attr.attr == "synchronize" + and isinstance(attr.value, ast.Attribute) + and attr.value.attr == "mps" + ): + found_device_sync = True + # event.synchronize() / Event.synchronize() — forbidden pattern. + if attr.attr == "synchronize" and isinstance(attr.value, ast.Name): + name = attr.value.id + if name.lower() in {"event", "start", "end"}: + forbidden_calls.append(f"{name}.synchronize()") + + assert found_device_sync, "event_timer must call torch.mps.synchronize()" + assert not forbidden_calls, ( + f"event_timer must NOT call per-event synchronize " + f"(pytorch#162872 deadlock); found: {forbidden_calls}" + ) + + +def test_mps_backend_event_timer_returns_positive_elapsed_ms(mps_backend) -> None: + torch = _torch() + x = torch.randn(64, 64, device="mps") + with mps_backend.event_timer() as t: + _ = x @ x + assert t.elapsed_ms >= 0.0 + + +def test_mps_backend_arch_info_returns_apple_silicon(mps_backend) -> None: + info = mps_backend.arch_info() + assert info.architecture == "Apple-Silicon" + assert info.backend == "mps" + assert info.tensor_core_generation is None + assert info.cuda_version == "" + # Compute capability is a CUDA concept; MPS uses (0, 0). + assert info.compute_capability == (0, 0) + # Apple chip name should be in the device name when sysctl is available + # (we only assert non-empty here so the test still passes in chrooted CI). + assert isinstance(info.name, str) + + +def test_mps_backend_flush_l2_is_noop_with_warning(mps_backend) -> None: + # Reset the module-level warning gate so we deterministically observe + # the warning even if a prior test already triggered it. + import gpucheck.backends.mps as mps_mod + + mps_mod._FLUSH_L2_WARNED = False + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + mps_backend.flush_l2() + assert any("L2" in str(w.message) for w in caught) + + +def test_mps_backend_mem_stats_has_required_keys(mps_backend) -> None: + stats = mps_backend.mem_stats() + assert "used" in stats + # driver_allocated and total are best-effort; just check keys exist. + assert "driver_allocated" in stats + assert "total" in stats + + +def test_mps_backend_device_count_is_one(mps_backend) -> None: + # MPS exposes a single logical device on every Apple Silicon machine. + assert mps_backend.device_count() == 1 diff --git a/tests/test_determinism.py b/tests/test_determinism.py new file mode 100644 index 0000000..460cee4 --- /dev/null +++ b/tests/test_determinism.py @@ -0,0 +1,189 @@ +"""Determinism sanitizer tests (Track D — D.3).""" + +from __future__ import annotations + +import random + +import pytest + +from gpucheck.sanitizers.determinism import ( + DeterminismError, + assert_deterministic, + requires_determinism, +) + + +def test_assert_deterministic_passes_when_output_is_deterministic() -> None: + def deterministic_fn() -> int: + return random.randint(0, 100) # noqa: S311 (test, not crypto) + + # Seeded properly, this returns the same int each call. + out = assert_deterministic(deterministic_fn, n=4, seed=7) + assert isinstance(out, int) + + +def test_assert_deterministic_raises_when_output_diverges() -> None: + counter = {"i": 0} + + def diverging_fn() -> int: + counter["i"] += 1 + return counter["i"] # 1, 2, 3, ... — not seeded by random + + with pytest.raises(DeterminismError, match="differs"): + assert_deterministic(diverging_fn, n=3, seed=0) + + +def test_assert_deterministic_rejects_n_lt_2() -> None: + with pytest.raises(ValueError, match="n >= 2"): + assert_deterministic(lambda: 1, n=1) + + +def test_assert_deterministic_returns_first_output() -> None: + def fn() -> str: + return "stable" + + out = assert_deterministic(fn, n=3) + assert out == "stable" + + +def test_requires_determinism_decorator_calls_fn_n_times() -> None: + counter = {"i": 0} + + @requires_determinism(n=4, seed=99) + def fn() -> int: + counter["i"] += 1 + # Reset randomness here makes this deterministic across calls + # because the decorator calls _seed_all between invocations. + return random.randint(0, 1_000_000) # noqa: S311 + + fn() + assert counter["i"] == 4 + + +def test_requires_determinism_propagates_determinism_error_from_unstable_fn() -> None: + counter = {"i": 0} + + @requires_determinism(n=2, seed=0) + def fn() -> int: + counter["i"] += 1 + return counter["i"] + + with pytest.raises(DeterminismError): + fn() + + +def test_assert_deterministic_handles_tuple_outputs() -> None: + def fn() -> tuple[int, int]: + return (random.randint(0, 100), random.randint(0, 100)) # noqa: S311 + + out = assert_deterministic(fn, n=3, seed=42) + assert isinstance(out, tuple) + assert len(out) == 2 + + +def test_assert_deterministic_handles_list_outputs_diverging() -> None: + counter = {"i": 0} + + def fn() -> list[int]: + counter["i"] += 1 + return [counter["i"]] + + with pytest.raises(DeterminismError): + assert_deterministic(fn, n=2, seed=0) + + +def test_assert_deterministic_with_torch_tensor_outputs() -> None: + """When torch is available, tensor equality goes through torch.equal.""" + torch = pytest.importorskip("torch") + + def fn() -> torch.Tensor: + return torch.randn(3, 3) # seeded => same tensor + + out = assert_deterministic(fn, n=3, seed=0) + assert out.shape == (3, 3) + + +# --------------------------------------------------------------------------- +# atol / rtol kwargs (review BLOCKER A2) +# --------------------------------------------------------------------------- + + +def test_assert_deterministic_default_is_byte_equal() -> None: + """Without atol/rtol, comparison is bit-exact via torch.equal — + a tensor that differs by 1 ULP still triggers DeterminismError. + """ + torch = pytest.importorskip("torch") + counter = {"i": 0} + + def drift_by_eps() -> torch.Tensor: + counter["i"] += 1 + # Different output each call: tiny perturbation but bit-different. + base = torch.zeros(2, 2) + return base + (counter["i"] * 1e-7) + + with pytest.raises(DeterminismError, match="byte-identical"): + assert_deterministic(drift_by_eps, n=2) + + +def test_assert_deterministic_atol_accepts_drift_within_tolerance() -> None: + """atol > drift => assert_deterministic passes. The contract that + review A2 demanded for MPS use cases. + """ + torch = pytest.importorskip("torch") + counter = {"i": 0} + + def drift_by_eps() -> torch.Tensor: + counter["i"] += 1 + base = torch.zeros(2, 2) + return base + (counter["i"] * 1e-7) + + # atol much larger than the per-call drift — should accept. + out = assert_deterministic(drift_by_eps, n=3, atol=1e-3) + assert out.shape == (2, 2) + + +def test_assert_deterministic_atol_rejects_drift_above_tolerance() -> None: + """atol < drift => assert_deterministic still fails (with the + new tolerance-mode error message). + """ + torch = pytest.importorskip("torch") + counter = {"i": 0} + + def drift_by_one() -> torch.Tensor: + counter["i"] += 1 + # Drift of 1.0 between calls — well above any reasonable atol. + return torch.zeros(2, 2) + counter["i"] + + with pytest.raises(DeterminismError, match="allclose"): + assert_deterministic(drift_by_one, n=2, atol=1e-3) + + +def test_assert_deterministic_rtol_accepts_relative_drift() -> None: + """rtol path mirrors atol — covers the rtol leg of allclose.""" + torch = pytest.importorskip("torch") + counter = {"i": 0} + + def drift_relative() -> torch.Tensor: + counter["i"] += 1 + # Drifts proportionally — rtol catches this, atol alone wouldn't. + return torch.full((2, 2), 1000.0) + counter["i"] * 1e-3 + + out = assert_deterministic(drift_relative, n=3, rtol=1e-2) + assert out.shape == (2, 2) + + +def test_requires_determinism_forwards_atol() -> None: + """The decorator must accept and forward atol so MPS users can + write `@requires_determinism(n=3, atol=1e-5)`. + """ + torch = pytest.importorskip("torch") + counter = {"i": 0} + + @requires_determinism(n=2, seed=0, atol=1e-3) + def drift_within() -> torch.Tensor: + counter["i"] += 1 + return torch.zeros(2, 2) + counter["i"] * 1e-7 + + out = drift_within() + assert out.shape == (2, 2) + assert counter["i"] == 2 diff --git a/tests/test_devices_mps.py b/tests/test_devices_mps.py new file mode 100644 index 0000000..01d1799 --- /dev/null +++ b/tests/test_devices_mps.py @@ -0,0 +1,63 @@ +"""@devices("mps") parametrization (Track A).""" + +from __future__ import annotations + +from gpucheck.decorators.devices import ( + _detect_devices, + _detect_mps_devices, + _is_device_available, + devices, +) + + +def _has_mps() -> bool: + try: + import torch + except ImportError: + return False + mps = getattr(torch.backends, "mps", None) + return bool(mps is not None and mps.is_available()) + + +def test_detect_mps_devices_when_available() -> None: + if _has_mps(): + assert _detect_mps_devices() == ["mps"] + else: + assert _detect_mps_devices() == [] + + +def test_detect_devices_includes_mps_when_available() -> None: + devs = _detect_devices() + if _has_mps(): + assert "mps" in devs + + +def test_is_device_available_mps_string() -> None: + if _has_mps(): + assert _is_device_available("mps") is True + else: + assert _is_device_available("mps") is False + + +# Explicit @devices("mps") usage — parametrizes the test even if MPS is +# unavailable (test gets pytest.mark.skip in that case). +@devices("mps") +def test_devices_decorator_passes_mps_string(device: str) -> None: + assert device == "mps" + if _has_mps(): + import torch + + x = torch.zeros(2, 2, device=device) + assert x.device.type == "mps" + + +@devices("cuda:0", "mps") +def test_devices_decorator_mixed_cuda_and_mps(device: str) -> None: + assert device in {"cuda:0", "mps"} + + +def test_all_keyword_includes_mps_on_apple_silicon() -> None: + """The 'all' keyword should expand to MPS on Apple Silicon.""" + devs = _detect_devices() + if _has_mps(): + assert "mps" in devs diff --git a/tests/test_fuzz_strides.py b/tests/test_fuzz_strides.py new file mode 100644 index 0000000..151dedf --- /dev/null +++ b/tests/test_fuzz_strides.py @@ -0,0 +1,173 @@ +"""Stride fuzzing — corpus + per-category contracts (Track B).""" + +from __future__ import annotations + +import pytest + +from gpucheck.fuzzing.strides import ( + CATEGORIES, + fuzz_strides, + fuzz_strides_for_category, +) + +torch = pytest.importorskip("torch") + + +def test_categories_are_seven_canonical() -> None: + assert CATEGORIES == ( + "row_major", + "column_major", + "broadcast", + "transpose", + "slice", + "non_contig", + "gather", + ) + + +def test_fuzz_strides_returns_all_categories_in_order() -> None: + out = fuzz_strides((64, 64), torch.float32, seed=0) + assert [c for c, _t in out] == list(CATEGORIES) + + +def test_fuzz_strides_each_tensor_has_correct_shape() -> None: + out = fuzz_strides((32, 32), torch.float32, seed=0) + for label, t in out: + assert t.shape == (32, 32), f"{label}: wrong shape {t.shape}" + + +def test_fuzz_strides_each_tensor_has_correct_dtype() -> None: + out = fuzz_strides((16, 16), torch.float16, seed=0) + for label, t in out: + assert t.dtype == torch.float16, f"{label}: wrong dtype {t.dtype}" + + +def test_row_major_is_contiguous() -> None: + t = fuzz_strides_for_category((64, 64), torch.float32, "row_major", seed=0) + assert t.is_contiguous() + + +def test_column_major_is_not_contiguous() -> None: + t = fuzz_strides_for_category((64, 32), torch.float32, "column_major", seed=0) + assert not t.is_contiguous() + + +def test_broadcast_has_stride_zero_on_last_dim() -> None: + t = fuzz_strides_for_category((4, 8, 16), torch.float32, "broadcast", seed=0) + # The last dim was expanded from size 1 -> 16, so stride is 0 there. + assert t.stride(-1) == 0 + assert t.shape == (4, 8, 16) + + +def test_transpose_is_not_contiguous() -> None: + t = fuzz_strides_for_category((4, 8, 16), torch.float32, "transpose", seed=0) + assert not t.is_contiguous() + # Transpose preserves shape because we transposed the LAST two dims of + # the contiguous (4, 16, 8) buffer. + assert t.shape == (4, 8, 16) + + +def test_slice_has_stride_two_on_each_dim() -> None: + t = fuzz_strides_for_category((8, 8), torch.float32, "slice", seed=0) + assert not t.is_contiguous() + assert t.shape == (8, 8) + + +def test_non_contig_2d_is_not_contiguous() -> None: + t = fuzz_strides_for_category((8, 8), torch.float32, "non_contig", seed=0) + assert not t.is_contiguous() + assert t.shape == (8, 8) + + +def test_gather_returns_contiguous_with_correct_shape() -> None: + t = fuzz_strides_for_category((4, 4), torch.float32, "gather", seed=0) + assert t.is_contiguous() + assert t.shape == (4, 4) + + +def test_unknown_category_raises() -> None: + with pytest.raises(ValueError, match="Unknown stride category"): + fuzz_strides_for_category((4,), torch.float32, "weird") + + +def test_fuzz_strides_with_n_caps_results() -> None: + out = fuzz_strides((4, 4), torch.float32, n=3, seed=0) + assert len(out) == 3 + + +def test_fuzz_strides_with_explicit_categories() -> None: + out = fuzz_strides( + (4, 4), torch.float32, + categories=("row_major", "transpose"), + seed=0, + ) + assert [c for c, _t in out] == ["row_major", "transpose"] + + +def test_fuzz_strides_invalid_category_raises() -> None: + with pytest.raises(ValueError, match="Unknown stride categories"): + fuzz_strides((4, 4), torch.float32, categories=("row_major", "weird")) + + +def test_fuzz_strides_seed_is_deterministic() -> None: + out1 = fuzz_strides((4, 4), torch.float32, seed=42) + out2 = fuzz_strides((4, 4), torch.float32, seed=42) + for (l1, t1), (l2, t2) in zip(out1, out2, strict=True): + assert l1 == l2 + assert torch.equal(t1, t2), f"seed-determinism broken for {l1}" + + +def test_1d_fallbacks_to_row_major_or_slice_correctly() -> None: + # 1D shape: column_major and transpose fall back to row_major, + # non_contig falls back to slice. + out = fuzz_strides((8,), torch.float32, seed=0) + # Just assert we got tensors of shape (8,) — the fallback semantics + # are documented in the docstring; this is a smoke check. + for _label, t in out: + assert t.shape == (8,) + + +def test_higher_rank_3d_smoke() -> None: + out = fuzz_strides((4, 8, 16), torch.float32, seed=0) + for _label, t in out: + assert t.shape == (4, 8, 16) + + +# --------------------------------------------------------------------------- +# Kebab-case alias acceptance (review BLOCKER A1) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("alias", "canonical"), + [ + ("row-major", "row_major"), + ("column-major", "column_major"), + ("broadcast-induced", "broadcast"), + ("non-contig", "non_contig"), + ("non-contiguous", "non_contig"), + ("contiguous-after-clone", "non_contig"), + ("gather-induced", "gather"), + ], +) +def test_fuzz_strides_for_category_accepts_kebab_alias( + alias: str, canonical: str, +) -> None: + """Both kebab-case (deprecated) and snake_case names must work.""" + with pytest.warns(DeprecationWarning, match=alias): + t_kebab = fuzz_strides_for_category((4, 4), torch.float32, alias, seed=0) + t_snake = fuzz_strides_for_category((4, 4), torch.float32, canonical, seed=0) + # Same seed, same shape, same dtype, alias-routed → same tensor. + assert torch.equal(t_kebab, t_snake) + + +def test_fuzz_strides_accepts_kebab_aliases_in_categories() -> None: + """Same alias acceptance via the bulk ``fuzz_strides`` entrypoint.""" + with pytest.warns(DeprecationWarning): + out = fuzz_strides( + (4, 4), torch.float32, + categories=("row-major", "broadcast-induced"), + seed=7, + ) + # The returned labels are the *canonical* names, not the kebab aliases. + assert [c for c, _t in out] == ["row_major", "broadcast"] diff --git a/tests/test_fuzz_strides_hypothesis.py b/tests/test_fuzz_strides_hypothesis.py new file mode 100644 index 0000000..3bbc94f --- /dev/null +++ b/tests/test_fuzz_strides_hypothesis.py @@ -0,0 +1,39 @@ +"""Stride fuzzing — Hypothesis StrideStrategy (Track B).""" + +from __future__ import annotations + +import pytest + +torch = pytest.importorskip("torch") +hypothesis = pytest.importorskip("hypothesis") + +from hypothesis import given, settings # noqa: E402 + +from gpucheck.fuzzing.strides import StrideStrategy # noqa: E402 + + +@settings(max_examples=20, deadline=None) +@given(t=StrideStrategy(shape=(8, 8), dtype=torch.float32)) +def test_stride_strategy_yields_tensor_of_target_shape(t) -> None: + assert t.shape == (8, 8) + assert t.dtype == torch.float32 + + +@settings(max_examples=10, deadline=None) +@given( + t=StrideStrategy( + shape=(4, 4), + dtype=torch.float32, + categories=("row_major",), + ), +) +def test_stride_strategy_with_single_category_only_returns_that_category(t) -> None: + # row_major is contiguous by construction. + assert t.is_contiguous() + assert t.shape == (4, 4) + + +def test_stride_strategy_unknown_dtype_does_not_crash_creation() -> None: + s = StrideStrategy(shape=(2, 2), dtype=torch.float16) + # Strategy creation must succeed; drawing also must not crash. + assert s is not None diff --git a/tests/test_mps_xfail.py b/tests/test_mps_xfail.py new file mode 100644 index 0000000..8b4bc8b --- /dev/null +++ b/tests/test_mps_xfail.py @@ -0,0 +1,90 @@ +"""MPS xfail registry config-loader (Track A).""" + +from __future__ import annotations + +from gpucheck import is_mps_xfailed, mps_xfail_list, register_mps_xfail +from gpucheck.assertions.tolerances import ( + apply_mps_xfail_config, + mps_xfail_from_config, + reset_mps_xfail, +) + +# The 12 entries we ship in pyproject.toml per SYNTHESIS §7. +_EXPECTED_XFAIL_OPS = { + "scaled_dot_product_attention.large", + "scaled_dot_product_attention.backward", + "layer_norm.backward.shape1", + "batch_norm.backward.channels_last", + "conv2d.large_channels", + "conv2d.backward.channels_last_format", + "F.linear.backward.bf16_3d_nobias_m5", + "softmax.large_attention", + "avg_pool2d.backward.channels_last", + "binary_ops.uint16_uint32_uint64", + "BCE_loss", + "matmul.backward.over_32K_elements", +} + + +def test_mps_xfail_from_config_extracts_ops_list() -> None: + cfg = { + "tool": { + "gpucheck": { + "mps": { + "xfail": {"ops": ["softmax.large_attention", "BCE_loss"]}, + } + } + } + } + assert mps_xfail_from_config(cfg) == {"softmax.large_attention", "BCE_loss"} + + +def test_mps_xfail_from_config_returns_none_for_empty() -> None: + assert mps_xfail_from_config({}) is None + assert mps_xfail_from_config({"tool": {"gpucheck": {}}}) is None + + +def test_apply_mps_xfail_replaces_existing_registry() -> None: + # Save the current registry so we can restore it (the plugin populated + # it from pyproject.toml at session start; other tests rely on that). + from gpucheck.assertions.tolerances import _mps_xfail_set + + saved = set(_mps_xfail_set) + try: + reset_mps_xfail() + register_mps_xfail("phantom.op") + assert is_mps_xfailed("phantom.op") + apply_mps_xfail_config({ + "tool": {"gpucheck": {"mps": {"xfail": {"ops": ["softmax.large_attention"]}}}} + }) + assert not is_mps_xfailed("phantom.op") + assert is_mps_xfailed("softmax.large_attention") + finally: + reset_mps_xfail() + register_mps_xfail(*saved) + + +def test_pyproject_xfail_block_loaded_at_session_start() -> None: + """The 12 SYNTHESIS §7 entries must populate the registry once the + plugin's pytest_configure has run (which it has, since we're running + inside pytest). + """ + actual = set(mps_xfail_list()) + missing = _EXPECTED_XFAIL_OPS - actual + assert not missing, ( + f"pyproject.toml [tool.gpucheck.mps.xfail] is missing entries: {missing}; " + f"the SYNTHESIS §7 living-document list must be populated." + ) + + +def test_register_mps_xfail_at_runtime() -> None: + register_mps_xfail("some.runtime.op") + try: + assert is_mps_xfailed("some.runtime.op") + finally: + # Don't pollute other tests. (reset_mps_xfail clears all entries + # including the pyproject-loaded ones, so we instead rebuild from + # config.) + from gpucheck.assertions.tolerances import _mps_xfail_set + + _mps_xfail_set.discard("some.runtime.op") diff --git a/tests/test_parametrize_gpu_strides.py b/tests/test_parametrize_gpu_strides.py new file mode 100644 index 0000000..6491c89 --- /dev/null +++ b/tests/test_parametrize_gpu_strides.py @@ -0,0 +1,48 @@ +"""parametrize_gpu(stride_categories=...) wiring (Track B).""" + +from __future__ import annotations + +import pytest + +from gpucheck.decorators.parametrize import parametrize_gpu + +torch = pytest.importorskip("torch") + + +@parametrize_gpu( + dtypes=("float32",), + shapes=((4, 4),), + devices=("cpu",), + stride_categories=("row_major", "transpose"), +) +def test_stride_categories_appear_in_signature(dtype, shape, device, stride_category) -> None: + assert stride_category in {"row_major", "transpose"} + assert shape == (4, 4) + assert device == "cpu" + + +def test_parametrize_gpu_rejects_unknown_stride_category() -> None: + with pytest.raises(ValueError, match="Unknown stride categories"): + parametrize_gpu( + dtypes=("float32",), + shapes=((4, 4),), + devices=("cpu",), + stride_categories=("row_major", "wat"), + ) + + +def test_parametrize_gpu_without_stride_categories_keeps_old_signature() -> None: + decorator = parametrize_gpu( + dtypes=("float32",), + shapes=((4, 4),), + devices=("cpu",), + ) + + # The decorator marker name should not contain stride_category. + @decorator + def _fake_test(dtype, shape, device) -> None: # noqa: ARG001 + pass + + # Inspect the param names attached by pytest.mark.parametrize: + marks = list(_fake_test.pytestmark) + assert any("stride_category" not in m.args[0] for m in marks) diff --git a/tests/test_plugin_tomli.py b/tests/test_plugin_tomli.py new file mode 100644 index 0000000..801654f --- /dev/null +++ b/tests/test_plugin_tomli.py @@ -0,0 +1,73 @@ +"""Plugin TOML loading dependency contract (review BLOCKER S1). + +Python 3.10 lacks ``tomllib`` in the stdlib; ``gpucheck.plugin`` +falls back to ``tomli``. This test pins the dependency: + +- On 3.10, ``tomli`` MUST be importable (i.e. declared in + ``[project.dependencies]`` with the right marker so a fresh + ``pip install gpucheck`` includes it). +- On 3.11+, the test skips because ``tomllib`` is the stdlib path. + +Without this guard, the catch-all in ``_load_pyproject_config`` +silently swallows the ImportError and the user's +``[tool.gpucheck.tolerances]`` and ``[tool.gpucheck.mps.xfail]`` +overlays no-op without warning — confidence-in-test failure. +""" + +from __future__ import annotations + +import importlib.util +import sys + +import pytest + + +@pytest.mark.skipif( + sys.version_info >= (3, 11), + reason="tomllib is in stdlib on Python 3.11+; tomli is only the 3.10 fallback", +) +def test_tomli_is_importable_on_python_310() -> None: + """``tomli`` must be a declared dependency on Python 3.10. + + The plugin's ``_load_pyproject_config`` does + ``import tomli as tomllib`` when the stdlib ``tomllib`` is absent. + If ``tomli`` isn't installed, the user's pyproject overlays + silently no-op (review BLOCKER S1). + """ + spec = importlib.util.find_spec("tomli") + assert spec is not None, ( + "tomli is required on Python 3.10 (the plugin's TOML fallback " + "path); declare it in pyproject.toml's [project.dependencies] " + "with `python_version < '3.11'` marker." + ) + + +def test_plugin_loader_reads_pyproject_overlay(tmp_path: object) -> None: + """End-to-end smoke test: a synthetic pyproject.toml with a + ``[tool.gpucheck.tolerances]`` block must apply on the current + Python version. This exercises whichever TOML backend the + interpreter resolves (stdlib on 3.11+, tomli on 3.10). + """ + from pathlib import Path + + from gpucheck.assertions.tolerances import ( + compute_tolerance, + reset_config_tolerances, + ) + from gpucheck.plugin import _load_pyproject_config + + rootpath = Path(str(tmp_path)) + (rootpath / "pyproject.toml").write_text( + """ +[tool.gpucheck.tolerances] +float16 = {atol = 9.99e-3, rtol = 9.99e-3} +""", + encoding="utf-8", + ) + try: + _load_pyproject_config(rootpath) + atol, rtol = compute_tolerance("float16") + assert atol == pytest.approx(9.99e-3) + assert rtol == pytest.approx(9.99e-3) + finally: + reset_config_tolerances() diff --git a/tests/test_race_cuda_home_allowlist.py b/tests/test_race_cuda_home_allowlist.py new file mode 100644 index 0000000..68fa84d --- /dev/null +++ b/tests/test_race_cuda_home_allowlist.py @@ -0,0 +1,127 @@ +"""TM-E1 mitigation: CUDA_HOME / CUDA_PATH allowlist (Track C).""" + +from __future__ import annotations + +import os +import warnings +from typing import TYPE_CHECKING + +from gpucheck.sanitizers.race import ( + _CUDA_HOME_ALLOWLIST, + _find_compute_sanitizer, + _is_allowed_cuda_home, +) + +if TYPE_CHECKING: + from pathlib import Path + + import pytest + + +def test_is_allowed_cuda_home_canonical_paths() -> None: + for prefix in _CUDA_HOME_ALLOWLIST: + assert _is_allowed_cuda_home(prefix) is True + assert _is_allowed_cuda_home(prefix + "/bin") is True + assert _is_allowed_cuda_home(prefix + "/12.2") is True + + +def test_is_allowed_cuda_home_rejects_lookalike_paths() -> None: + # Trailing characters must NOT match the prefix. + assert _is_allowed_cuda_home("/usr/local/cuda-evil") is False + assert _is_allowed_cuda_home("/opt/nvidia/cudawat") is False + assert _is_allowed_cuda_home("/opt") is False + assert _is_allowed_cuda_home("/tmp/attacker") is False + + +def test_find_compute_sanitizer_returns_none_when_path_lookup_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + # Ensure shutil.which fails (point PATH at an empty dir). + monkeypatch.setenv("PATH", str(tmp_path)) + monkeypatch.delenv("CUDA_HOME", raising=False) + monkeypatch.delenv("CUDA_PATH", raising=False) + + assert _find_compute_sanitizer() is None + + +def test_find_compute_sanitizer_rejects_outside_allowlist( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + """A CUDA_HOME outside the allowlist must be ignored AND emit a warning.""" + monkeypatch.setenv("PATH", str(tmp_path)) # neutralize shutil.which path + + fake_cuda = tmp_path / "fake_cuda" + (fake_cuda / "bin").mkdir(parents=True) + binary = fake_cuda / "bin" / "compute-sanitizer" + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o755) + + monkeypatch.setenv("CUDA_HOME", str(fake_cuda)) + monkeypatch.delenv("CUDA_PATH", raising=False) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = _find_compute_sanitizer() + + assert result is None, ( + "fake CUDA_HOME outside allowlist must NOT yield a sanitizer path" + ) + assert any("allowlist" in str(w.message) for w in caught), ( + "expected a RuntimeWarning explaining the allowlist rejection" + ) + + +def test_find_compute_sanitizer_accepts_inside_allowlist( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + """If CUDA_HOME points inside the allowlist AND a binary exists there, return it. + + We can't actually create files at /usr/local/cuda in CI; instead, we + monkeypatch _CUDA_HOME_ALLOWLIST to include tmp_path and verify the + code path returns the binary when the rest of the conditions hold. + """ + monkeypatch.setenv("PATH", str(tmp_path / "no_path_here")) + + real_cuda = tmp_path / "real_cuda" + (real_cuda / "bin").mkdir(parents=True) + binary = real_cuda / "bin" / "compute-sanitizer" + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o755) + + monkeypatch.setattr( + "gpucheck.sanitizers.race._CUDA_HOME_ALLOWLIST", + (str(real_cuda.resolve()),), + ) + monkeypatch.setenv("CUDA_HOME", str(real_cuda)) + monkeypatch.delenv("CUDA_PATH", raising=False) + + result = _find_compute_sanitizer() + assert result == os.path.join(str(real_cuda.resolve()), "bin", "compute-sanitizer") + + +def test_find_compute_sanitizer_resolves_symlink_before_allowlist( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + """A symlink pointing OUTSIDE the allowlist must be rejected. + + This guards against the obvious attack: + ln -s /tmp/attacker /usr/local/cuda + """ + monkeypatch.setenv("PATH", str(tmp_path / "nope")) + + attacker = tmp_path / "attacker" + (attacker / "bin").mkdir(parents=True) + (attacker / "bin" / "compute-sanitizer").write_text("#!/bin/sh\nexit 0\n") + (attacker / "bin" / "compute-sanitizer").chmod(0o755) + + symlink_at_canonical = tmp_path / "symlinked_cuda" + symlink_at_canonical.symlink_to(attacker) + + monkeypatch.setenv("CUDA_HOME", str(symlink_at_canonical)) + monkeypatch.delenv("CUDA_PATH", raising=False) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = _find_compute_sanitizer() + assert result is None + assert any("allowlist" in str(w.message) for w in caught) diff --git a/tests/test_reporting_ci.py b/tests/test_reporting_ci.py new file mode 100644 index 0000000..022adc5 --- /dev/null +++ b/tests/test_reporting_ci.py @@ -0,0 +1,126 @@ +"""CI reporting tests (Track D — D.1).""" + +from __future__ import annotations + +import io +import sys +from typing import TYPE_CHECKING + +from gpucheck.reporting.ci import ( + emit_github_annotations, + generate_pr_comment, + write_junit_xml, +) +from gpucheck.reporting.console import TestResult + +if TYPE_CHECKING: + from pathlib import Path + + +def test_emit_github_annotations_writes_error_lines(monkeypatch) -> None: + monkeypatch.setenv("GITHUB_ACTIONS", "1") + captured = io.StringIO() + monkeypatch.setattr(sys, "stdout", captured) + results = [ + TestResult( + name="tests/test_x.py::test_y", status="failed", + duration=0.1, message="AssertionError\nexpected != actual", + file="tests/test_x.py", line=42, + ), + ] + emit_github_annotations(results) + out = captured.getvalue() + assert "::error" in out + assert "file=tests/test_x.py" in out + assert "line=42" in out + assert "%0A" in out # newline encoded in annotation message + + +def test_emit_github_annotations_skipped_warning(monkeypatch) -> None: + monkeypatch.setenv("GITHUB_ACTIONS", "1") + captured = io.StringIO() + monkeypatch.setattr(sys, "stdout", captured) + results = [TestResult(name="t1", status="skipped", message="no gpu")] + emit_github_annotations(results) + assert "::warning" in captured.getvalue() + + +def test_emit_github_annotations_passed_is_silent(monkeypatch) -> None: + monkeypatch.setenv("GITHUB_ACTIONS", "1") + captured = io.StringIO() + monkeypatch.setattr(sys, "stdout", captured) + emit_github_annotations([TestResult(name="t1", status="passed")]) + assert captured.getvalue() == "" + + +def test_emit_github_annotations_no_op_outside_actions(monkeypatch) -> None: + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + captured = io.StringIO() + monkeypatch.setattr(sys, "stdout", captured) + emit_github_annotations([TestResult(name="t1", status="failed", message="boom")]) + assert captured.getvalue() == "" + + +def test_write_junit_xml_emits_valid_xml(tmp_path: Path) -> None: + out = tmp_path / "junit.xml" + results = [ + TestResult(name="t1", status="passed", duration=0.1), + TestResult(name="t2", status="failed", duration=0.2, message="boom"), + TestResult(name="t3", status="skipped", duration=0.0, message="no gpu"), + TestResult(name="t4", status="error", duration=0.05, message="crashed"), + ] + path = write_junit_xml(results, output_path=out, suite_name="gpucheck") + + assert path == out + text = out.read_text() + assert ' None: + diff = { + "benchmarks": [ + { + "name": "matmul", + "baseline_median_ms": 1.0, + "current_median_ms": 1.6, + "delta_pct": 60.0, + "status": "regression", + }, + { + "name": "softmax", + "baseline_median_ms": 0.5, + "current_median_ms": 0.51, + "delta_pct": 2.0, + "status": "ok", + }, + {"name": "newkern", "current_median_ms": 0.3, "status": "new"}, + {"name": "removed", "baseline_median_ms": 0.7, "status": "removed"}, + ], + "test_changes": [ + {"name": "test_x", "was": "passed", "now": "failed"}, + ], + } + body = generate_pr_comment(diff) + assert "## gpucheck Benchmark Comparison" in body + assert "matmul" in body + assert "softmax" in body + assert "newkern" in body + assert "removed" in body + assert "test_x" in body + assert ":red_circle:" in body + assert ":green_circle:" in body + assert ":new:" in body + + +def test_generate_pr_comment_empty_diff_returns_friendly_message() -> None: + body = generate_pr_comment({"benchmarks": [], "test_changes": []}) + assert "No changes detected." in body diff --git a/tests/test_reporting_console.py b/tests/test_reporting_console.py new file mode 100644 index 0000000..7768164 --- /dev/null +++ b/tests/test_reporting_console.py @@ -0,0 +1,115 @@ +"""Console reporter tests (Track D — D.1).""" + +from __future__ import annotations + +import io + +from rich.console import Console + +from gpucheck.reporting.console import ( + BenchmarkEntry, + ConsoleReporter, + MemoryEntry, + TestResult, +) + + +def _new_reporter() -> tuple[ConsoleReporter, io.StringIO]: + buf = io.StringIO() + console = Console(file=buf, force_terminal=False, width=120) + return ConsoleReporter(console=console), buf + + +def test_console_reporter_constructs_with_explicit_console() -> None: + reporter, _ = _new_reporter() + assert reporter is not None + + +def test_gpu_info_panel_renders_keys_and_values() -> None: + reporter, buf = _new_reporter() + reporter.gpu_info_panel({"Device": "GTX 1650", "Compute": "7.5"}) + out = buf.getvalue() + assert "Device" in out + assert "GTX 1650" in out + assert "Compute" in out + assert "7.5" in out + + +def test_test_summary_includes_pass_fail_skip_counts() -> None: + reporter, buf = _new_reporter() + results = [ + TestResult(name="t1", status="passed", duration=0.1), + TestResult(name="t2", status="failed", duration=0.2, message="boom"), + TestResult(name="t3", status="skipped", duration=0.0, message="no gpu"), + ] + reporter.test_summary(results) + out = buf.getvalue() + assert "PASSED" in out + assert "FAILED" in out + assert "SKIPPED" in out + assert "1 passed" in out + assert "1 failed" in out + assert "1 skipped" in out + + +def test_benchmark_table_includes_kernel_and_throughput() -> None: + reporter, buf = _new_reporter() + entries = [ + BenchmarkEntry(name="matmul", times=[0.001, 0.0011, 0.0009]), + ] + reporter.benchmark_table(entries) + out = buf.getvalue() + assert "matmul" in out + assert "Median" in out + + +def test_memory_summary_shows_leak_status_red_for_leaks() -> None: + reporter, buf = _new_reporter() + entries = [ + MemoryEntry(name="leaky", peak_mb=10.5, leaked_mb=2.0, allocations=4), + MemoryEntry(name="clean", peak_mb=5.0, leaked_mb=0.0, allocations=2), + ] + reporter.memory_summary(entries) + out = buf.getvalue() + assert "leaky" in out + assert "clean" in out + assert "2.00" in out # 2.0 MB leaked + + +def test_error_detail_renders_name_and_traceback() -> None: + reporter, buf = _new_reporter() + reporter.error_detail("test_x", "AssertionError: nope", traceback="line1\nline2") + out = buf.getvalue() + assert "test_x" in out + assert "AssertionError" in out + + +def test_console_reporter_uses_stderr_in_ci_environment(monkeypatch) -> None: + """When GITHUB_ACTIONS=1, the reporter writes to stderr by default.""" + monkeypatch.setenv("GITHUB_ACTIONS", "1") + monkeypatch.delenv("CI", raising=False) + reporter = ConsoleReporter() + # Internal: assert the file is sys.stderr (Rich's Console exposes file). + import sys + assert reporter._console.file is sys.stderr # noqa: SLF001 + + +def test_console_reporter_with_file_kwarg() -> None: + """Constructing with file= takes precedence over CI/GITHUB_ACTIONS env.""" + buf = io.StringIO() + reporter = ConsoleReporter(file=buf) + reporter.gpu_info_panel({"x": "y"}) + assert "x" in buf.getvalue() + + +def test_benchmark_entry_throughput_handles_zero_times() -> None: + entry = BenchmarkEntry(name="empty", times=[]) + assert entry.median == 0.0 + assert entry.std == 0.0 + assert entry.throughput == 0.0 + + +def test_benchmark_entry_std_with_two_samples() -> None: + entry = BenchmarkEntry(name="k", times=[1.0, 2.0]) + assert entry.median == 1.5 + assert entry.std > 0 # statistics.stdev requires len >= 2 diff --git a/tests/test_reporting_html.py b/tests/test_reporting_html.py new file mode 100644 index 0000000..20cf43e --- /dev/null +++ b/tests/test_reporting_html.py @@ -0,0 +1,157 @@ +"""HTML dashboard tests (Track D — D.2).""" + +from __future__ import annotations + +import json +from html.parser import HTMLParser +from typing import TYPE_CHECKING + +from gpucheck.reporting.html import HTMLReporter + +if TYPE_CHECKING: + from pathlib import Path + + +def _write_sample_json(path: Path) -> None: + payload = { + "schema_version": 1, + "timestamp": "2026-05-01T12:00:00", + "gpu_info": {"name": "GTX 1650", "compute": "7.5"}, + "test_results": [ + {"name": "test_pass", "status": "passed", "duration": 0.1, "message": ""}, + {"name": "test_fail", "status": "failed", "duration": 0.2, + "message": "AssertionError: nope"}, + {"name": "test_skip", "status": "skipped", "duration": 0.0, + "message": "no gpu"}, + ], + "benchmarks": [ + {"name": "matmul", "median_ms": 1.0, "std_ms": 0.05, "samples": 100, "times": []}, + {"name": "softmax", "median_ms": 0.5, "std_ms": 0.02, "samples": 100, "times": []}, + ], + "memory": [ + {"name": "test_pass", "peak_mb": 10.0, "leaked_mb": 0.0, "allocations": 4}, + {"name": "test_fail", "peak_mb": 20.0, "leaked_mb": 2.5, "allocations": 8}, + ], + } + path.write_text(json.dumps(payload), encoding="utf-8") + + +class _TagCounter(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.tags: list[str] = [] + + def handle_starttag(self, tag: str, attrs) -> None: + self.tags.append(tag) + + +def test_html_reporter_writes_self_contained_html(tmp_path: Path) -> None: + json_path = tmp_path / "results.json" + _write_sample_json(json_path) + out = tmp_path / "dashboard.html" + + HTMLReporter(json_path).render(out) + assert out.exists() + text = out.read_text(encoding="utf-8") + assert text.startswith("") + # No external assets (no with http: or https:). + assert 'href="http' not in text + assert 'src="http' not in text + + +def test_html_reporter_contains_summary_counts(tmp_path: Path) -> None: + json_path = tmp_path / "results.json" + _write_sample_json(json_path) + out = tmp_path / "dashboard.html" + HTMLReporter(json_path).render(out) + text = out.read_text() + # 1 passed, 1 failed, 1 skipped — summary cards must surface those. + assert ">1<" in text # one of the cards renders 1 + # Test names appear in the table. + assert "test_pass" in text + assert "test_fail" in text + + +def test_html_reporter_includes_benchmark_table_with_kernel_names(tmp_path: Path) -> None: + json_path = tmp_path / "results.json" + _write_sample_json(json_path) + out = tmp_path / "dashboard.html" + HTMLReporter(json_path).render(out) + text = out.read_text() + assert "matmul" in text + assert "softmax" in text + assert " None: + json_path = tmp_path / "results.json" + _write_sample_json(json_path) + out = tmp_path / "dashboard.html" + diff = { + "benchmarks": [ + { + "name": "matmul", "baseline_median_ms": 1.0, + "current_median_ms": 1.6, "delta_pct": 60.0, + "status": "regression", + }, + ], + } + HTMLReporter(json_path, comparison=diff).render(out) + text = out.read_text() + assert "Comparison vs Baseline" in text + assert "REGRESSION" in text + + +def test_html_reporter_creates_parent_dir(tmp_path: Path) -> None: + json_path = tmp_path / "results.json" + _write_sample_json(json_path) + out = tmp_path / "deep" / "nested" / "dashboard.html" + HTMLReporter(json_path).render(out) + assert out.exists() + + +def test_html_reporter_html_is_well_formed(tmp_path: Path) -> None: + """HTMLParser tolerates malformed HTML, but should at least parse and + open balanced top-level tags (html, body).""" + json_path = tmp_path / "results.json" + _write_sample_json(json_path) + out = tmp_path / "dashboard.html" + HTMLReporter(json_path).render(out) + text = out.read_text() + counter = _TagCounter() + counter.feed(text) + assert "html" in counter.tags + assert "body" in counter.tags + assert "table" in counter.tags + + +def test_html_reporter_handles_empty_data(tmp_path: Path) -> None: + """No tests / no benchmarks must not crash the renderer.""" + json_path = tmp_path / "empty.json" + json_path.write_text(json.dumps({ + "schema_version": 1, "timestamp": "", "gpu_info": {}, + "test_results": [], "benchmarks": [], "memory": [], + })) + out = tmp_path / "dash.html" + HTMLReporter(json_path).render(out) + text = out.read_text() + assert "Summary" in text + # Test Results / Benchmarks / Memory sections are skipped when empty. + assert "Test Results" not in text + assert "Benchmarks" not in text + + +def test_html_reporter_escapes_html_in_messages(tmp_path: Path) -> None: + json_path = tmp_path / "results.json" + json_path.write_text(json.dumps({ + "test_results": [ + {"name": "test_x", "status": "failed", "duration": 0.1, + "message": ""}, + ], + "benchmarks": [], "memory": [], "gpu_info": {}, "timestamp": "", + })) + out = tmp_path / "dash.html" + HTMLReporter(json_path).render(out) + text = out.read_text() + assert "